xref: /petsc/src/dm/impls/plex/plexcreate.c (revision c22d3578198e541d0b2f39fc5e16a721df30154e)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h>    /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>          /*I   "petscdmplex.h"   I*/
4 #include <petscsf.h>
5 #include <petscdmplextransform.h>
6 #include <petsc/private/kernels/blockmatmult.h>
7 #include <petsc/private/kernels/blockinvert.h>
8 
9 PetscLogEvent DMPLEX_CreateFromFile, DMPLEX_BuildFromCellList, DMPLEX_BuildCoordinatesFromCellList;
10 
11 /* External function declarations here */
12 static PetscErrorCode DMInitialize_Plex(DM dm);
13 
14 /* This copies internal things in the Plex structure that we generally want when making a new, related Plex */
15 PetscErrorCode DMPlexCopy_Internal(DM dmin, PetscBool copyPeriodicity, PetscBool copyOverlap, DM dmout)
16 {
17   const PetscReal         *maxCell, *Lstart, *L;
18   PetscBool                dist;
19   DMPlexReorderDefaultFlag reorder;
20 
21   PetscFunctionBegin;
22   if (copyPeriodicity) {
23     PetscCall(DMGetPeriodicity(dmin, &maxCell, &Lstart, &L));
24     PetscCall(DMSetPeriodicity(dmout, maxCell,  Lstart,  L));
25   }
26   PetscCall(DMPlexDistributeGetDefault(dmin, &dist));
27   PetscCall(DMPlexDistributeSetDefault(dmout, dist));
28   PetscCall(DMPlexReorderGetDefault(dmin, &reorder));
29   PetscCall(DMPlexReorderSetDefault(dmout, reorder));
30   ((DM_Plex *) dmout->data)->useHashLocation  = ((DM_Plex *) dmin->data)->useHashLocation;
31   if (copyOverlap) PetscCall(DMPlexSetOverlap_Plex(dmout, dmin, 0));
32   PetscFunctionReturn(0);
33 }
34 
35 /* Replace dm with the contents of ndm, and then destroy ndm
36    - Share the DM_Plex structure
37    - Share the coordinates
38    - Share the SF
39 */
40 static PetscErrorCode DMPlexReplace_Static(DM dm, DM *ndm)
41 {
42   PetscSF          sf;
43   DM               dmNew = *ndm, coordDM, coarseDM;
44   Vec              coords;
45   const PetscReal *maxCell, *Lstart, *L;
46   PetscInt         dim, cdim;
47 
48   PetscFunctionBegin;
49   if (dm == dmNew) {
50     PetscCall(DMDestroy(ndm));
51     PetscFunctionReturn(0);
52   }
53   dm->setupcalled = dmNew->setupcalled;
54   PetscCall(DMGetDimension(dmNew, &dim));
55   PetscCall(DMSetDimension(dm, dim));
56   PetscCall(DMGetCoordinateDim(dmNew, &cdim));
57   PetscCall(DMSetCoordinateDim(dm, cdim));
58   PetscCall(DMGetPointSF(dmNew, &sf));
59   PetscCall(DMSetPointSF(dm, sf));
60   PetscCall(DMGetCoordinateDM(dmNew, &coordDM));
61   PetscCall(DMGetCoordinatesLocal(dmNew, &coords));
62   PetscCall(DMSetCoordinateDM(dm, coordDM));
63   PetscCall(DMSetCoordinatesLocal(dm, coords));
64   PetscCall(DMGetCellCoordinateDM(dmNew, &coordDM));
65   PetscCall(DMGetCellCoordinatesLocal(dmNew, &coords));
66   PetscCall(DMSetCellCoordinateDM(dm, coordDM));
67   PetscCall(DMSetCellCoordinatesLocal(dm, coords));
68   /* Do not want to create the coordinate field if it does not already exist, so do not call DMGetCoordinateField() */
69   PetscCall(DMFieldDestroy(&dm->coordinates[0].field));
70   dm->coordinates[0].field = dmNew->coordinates[0].field;
71   ((DM_Plex *) dmNew->data)->coordFunc = ((DM_Plex *) dm->data)->coordFunc;
72   PetscCall(DMGetPeriodicity(dmNew, &maxCell, &Lstart, &L));
73   PetscCall(DMSetPeriodicity(dm,     maxCell,  Lstart,  L));
74   PetscCall(DMDestroy_Plex(dm));
75   PetscCall(DMInitialize_Plex(dm));
76   dm->data = dmNew->data;
77   ((DM_Plex *) dmNew->data)->refct++;
78   PetscCall(DMDestroyLabelLinkList_Internal(dm));
79   PetscCall(DMCopyLabels(dmNew, dm, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL));
80   PetscCall(DMGetCoarseDM(dmNew,&coarseDM));
81   PetscCall(DMSetCoarseDM(dm,coarseDM));
82   PetscCall(DMDestroy(ndm));
83   PetscFunctionReturn(0);
84 }
85 
86 /* Swap dm with the contents of dmNew
87    - Swap the DM_Plex structure
88    - Swap the coordinates
89    - Swap the point PetscSF
90 */
91 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
92 {
93   DM              coordDMA, coordDMB;
94   Vec             coordsA,  coordsB;
95   PetscSF         sfA,      sfB;
96   DMField         fieldTmp;
97   void            *tmp;
98   DMLabelLink     listTmp;
99   DMLabel         depthTmp;
100   PetscInt        tmpI;
101 
102   PetscFunctionBegin;
103   if (dmA == dmB) PetscFunctionReturn(0);
104   PetscCall(DMGetPointSF(dmA, &sfA));
105   PetscCall(DMGetPointSF(dmB, &sfB));
106   PetscCall(PetscObjectReference((PetscObject) sfA));
107   PetscCall(DMSetPointSF(dmA, sfB));
108   PetscCall(DMSetPointSF(dmB, sfA));
109   PetscCall(PetscObjectDereference((PetscObject) sfA));
110 
111   PetscCall(DMGetCoordinateDM(dmA, &coordDMA));
112   PetscCall(DMGetCoordinateDM(dmB, &coordDMB));
113   PetscCall(PetscObjectReference((PetscObject) coordDMA));
114   PetscCall(DMSetCoordinateDM(dmA, coordDMB));
115   PetscCall(DMSetCoordinateDM(dmB, coordDMA));
116   PetscCall(PetscObjectDereference((PetscObject) coordDMA));
117 
118   PetscCall(DMGetCoordinatesLocal(dmA, &coordsA));
119   PetscCall(DMGetCoordinatesLocal(dmB, &coordsB));
120   PetscCall(PetscObjectReference((PetscObject) coordsA));
121   PetscCall(DMSetCoordinatesLocal(dmA, coordsB));
122   PetscCall(DMSetCoordinatesLocal(dmB, coordsA));
123   PetscCall(PetscObjectDereference((PetscObject) coordsA));
124 
125   PetscCall(DMGetCellCoordinateDM(dmA, &coordDMA));
126   PetscCall(DMGetCellCoordinateDM(dmB, &coordDMB));
127   PetscCall(PetscObjectReference((PetscObject) coordDMA));
128   PetscCall(DMSetCellCoordinateDM(dmA, coordDMB));
129   PetscCall(DMSetCellCoordinateDM(dmB, coordDMA));
130   PetscCall(PetscObjectDereference((PetscObject) coordDMA));
131 
132   PetscCall(DMGetCellCoordinatesLocal(dmA, &coordsA));
133   PetscCall(DMGetCellCoordinatesLocal(dmB, &coordsB));
134   PetscCall(PetscObjectReference((PetscObject) coordsA));
135   PetscCall(DMSetCellCoordinatesLocal(dmA, coordsB));
136   PetscCall(DMSetCellCoordinatesLocal(dmB, coordsA));
137   PetscCall(PetscObjectDereference((PetscObject) coordsA));
138 
139   fieldTmp                  = dmA->coordinates[0].field;
140   dmA->coordinates[0].field = dmB->coordinates[0].field;
141   dmB->coordinates[0].field = fieldTmp;
142   fieldTmp                  = dmA->coordinates[1].field;
143   dmA->coordinates[1].field = dmB->coordinates[1].field;
144   dmB->coordinates[1].field = fieldTmp;
145   tmp       = dmA->data;
146   dmA->data = dmB->data;
147   dmB->data = tmp;
148   listTmp   = dmA->labels;
149   dmA->labels = dmB->labels;
150   dmB->labels = listTmp;
151   depthTmp  = dmA->depthLabel;
152   dmA->depthLabel = dmB->depthLabel;
153   dmB->depthLabel = depthTmp;
154   depthTmp  = dmA->celltypeLabel;
155   dmA->celltypeLabel = dmB->celltypeLabel;
156   dmB->celltypeLabel = depthTmp;
157   tmpI         = dmA->levelup;
158   dmA->levelup = dmB->levelup;
159   dmB->levelup = tmpI;
160   PetscFunctionReturn(0);
161 }
162 
163 static PetscErrorCode DMPlexInterpolateInPlace_Internal(DM dm)
164 {
165   DM             idm;
166 
167   PetscFunctionBegin;
168   PetscCall(DMPlexInterpolate(dm, &idm));
169   PetscCall(DMPlexCopyCoordinates(dm, idm));
170   PetscCall(DMPlexReplace_Static(dm, &idm));
171   PetscFunctionReturn(0);
172 }
173 
174 /*@C
175   DMPlexCreateCoordinateSpace - Creates a finite element space for the coordinates
176 
177   Collective
178 
179   Input Parameters:
180 + DM        - The DM
181 . degree    - The degree of the finite element or PETSC_DECIDE
182 - coordFunc - An optional function to map new points from refinement to the surface
183 
184   Level: advanced
185 
186 .seealso: `PetscFECreateLagrange()`, `DMGetCoordinateDM()`
187 @*/
188 PetscErrorCode DMPlexCreateCoordinateSpace(DM dm, PetscInt degree, PetscPointFunc coordFunc)
189 {
190   DM_Plex      *mesh = (DM_Plex *) dm->data;
191   DM            cdm;
192   PetscDS       cds;
193   PetscFE       fe;
194   PetscClassId  id;
195 
196   PetscFunctionBegin;
197   PetscCall(DMGetCoordinateDM(dm, &cdm));
198   PetscCall(DMGetDS(cdm, &cds));
199   PetscCall(PetscDSGetDiscretization(cds, 0, (PetscObject *) &fe));
200   PetscCall(PetscObjectGetClassId((PetscObject) fe, &id));
201   if (id != PETSCFE_CLASSID) {
202     PetscBool      simplex;
203     PetscInt       dim, dE, qorder;
204 
205     PetscCall(DMGetDimension(dm, &dim));
206     PetscCall(DMGetCoordinateDim(dm, &dE));
207     qorder = degree;
208     PetscObjectOptionsBegin((PetscObject) cdm);
209     PetscCall(PetscOptionsBoundedInt("-coord_dm_default_quadrature_order", "Quadrature order is one less than quadrature points per edge", "DMPlexCreateCoordinateSpace", qorder, &qorder, NULL, 0));
210     PetscOptionsEnd();
211     if (degree == PETSC_DECIDE) fe = NULL;
212     else {
213       PetscCall(DMPlexIsSimplex(dm, &simplex));
214       PetscCall(PetscFECreateLagrange(PETSC_COMM_SELF, dim, dE, simplex, degree, qorder, &fe));
215     }
216     PetscCall(DMProjectCoordinates(dm, fe));
217     PetscCall(PetscFEDestroy(&fe));
218   }
219   mesh->coordFunc = coordFunc;
220   PetscFunctionReturn(0);
221 }
222 
223 /*@
224   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
225 
226   Collective
227 
228   Input Parameters:
229 + comm - The communicator for the DM object
230 . dim - The spatial dimension
231 . simplex - Flag for simplicial cells, otherwise they are tensor product cells
232 . interpolate - Flag to create intermediate mesh pieces (edges, faces)
233 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
234 
235   Output Parameter:
236 . dm - The DM object
237 
238   Level: beginner
239 
240 .seealso: `DMSetType()`, `DMCreate()`
241 @*/
242 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscReal refinementLimit, DM *newdm)
243 {
244   DM             dm;
245   PetscMPIInt    rank;
246 
247   PetscFunctionBegin;
248   PetscCall(DMCreate(comm, &dm));
249   PetscCall(DMSetType(dm, DMPLEX));
250   PetscCall(DMSetDimension(dm, dim));
251   PetscCallMPI(MPI_Comm_rank(comm, &rank));
252   switch (dim) {
253   case 2:
254     if (simplex) PetscCall(PetscObjectSetName((PetscObject) dm, "triangular"));
255     else         PetscCall(PetscObjectSetName((PetscObject) dm, "quadrilateral"));
256     break;
257   case 3:
258     if (simplex) PetscCall(PetscObjectSetName((PetscObject) dm, "tetrahedral"));
259     else         PetscCall(PetscObjectSetName((PetscObject) dm, "hexahedral"));
260     break;
261   default:
262     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
263   }
264   if (rank) {
265     PetscInt numPoints[2] = {0, 0};
266     PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL));
267   } else {
268     switch (dim) {
269     case 2:
270       if (simplex) {
271         PetscInt    numPoints[2]        = {4, 2};
272         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
273         PetscInt    cones[6]            = {2, 3, 4,  5, 4, 3};
274         PetscInt    coneOrientations[6] = {0, 0, 0,  0, 0, 0};
275         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
276 
277         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
278       } else {
279         PetscInt    numPoints[2]        = {6, 2};
280         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
281         PetscInt    cones[8]            = {2, 3, 4, 5,  3, 6, 7, 4};
282         PetscInt    coneOrientations[8] = {0, 0, 0, 0,  0, 0, 0, 0};
283         PetscScalar vertexCoords[12]    = {-1.0, -0.5,  0.0, -0.5,  0.0, 0.5,  -1.0, 0.5,  1.0, -0.5,  1.0, 0.5};
284 
285         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
286       }
287       break;
288     case 3:
289       if (simplex) {
290         PetscInt    numPoints[2]        = {5, 2};
291         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
292         PetscInt    cones[8]            = {4, 3, 5, 2,  5, 3, 4, 6};
293         PetscInt    coneOrientations[8] = {0, 0, 0, 0,  0, 0, 0, 0};
294         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0,  0.0, -1.0, 0.0,  0.0, 0.0, 1.0,  0.0, 1.0, 0.0,  1.0, 0.0, 0.0};
295 
296         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
297       } else {
298         PetscInt    numPoints[2]         = {12, 2};
299         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
300         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9,  5, 4, 10, 11, 7, 12, 13, 8};
301         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0,  0, 0,  0,  0, 0,  0,  0, 0};
302         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5,  -1.0,  0.5, -0.5,  0.0,  0.5, -0.5,   0.0, -0.5, -0.5,
303                                             -1.0, -0.5,  0.5,   0.0, -0.5,  0.5,  0.0,  0.5,  0.5,  -1.0,  0.5,  0.5,
304                                              1.0,  0.5, -0.5,   1.0, -0.5, -0.5,  1.0, -0.5,  0.5,   1.0,  0.5,  0.5};
305 
306         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
307       }
308       break;
309     default:
310       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
311     }
312   }
313   *newdm = dm;
314   if (refinementLimit > 0.0) {
315     DM rdm;
316     const char *name;
317 
318     PetscCall(DMPlexSetRefinementUniform(*newdm, PETSC_FALSE));
319     PetscCall(DMPlexSetRefinementLimit(*newdm, refinementLimit));
320     PetscCall(DMRefine(*newdm, comm, &rdm));
321     PetscCall(PetscObjectGetName((PetscObject) *newdm, &name));
322     PetscCall(PetscObjectSetName((PetscObject)    rdm,  name));
323     PetscCall(DMDestroy(newdm));
324     *newdm = rdm;
325   }
326   if (interpolate) {
327     DM idm;
328 
329     PetscCall(DMPlexInterpolate(*newdm, &idm));
330     PetscCall(DMDestroy(newdm));
331     *newdm = idm;
332   }
333   PetscFunctionReturn(0);
334 }
335 
336 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
337 {
338   const PetscInt numVertices    = 2;
339   PetscInt       markerRight    = 1;
340   PetscInt       markerLeft     = 1;
341   PetscBool      markerSeparate = PETSC_FALSE;
342   Vec            coordinates;
343   PetscSection   coordSection;
344   PetscScalar   *coords;
345   PetscInt       coordSize;
346   PetscMPIInt    rank;
347   PetscInt       cdim = 1, v;
348 
349   PetscFunctionBegin;
350   PetscCall(PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
351   if (markerSeparate) {
352     markerRight  = 2;
353     markerLeft   = 1;
354   }
355   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
356   if (rank == 0) {
357     PetscCall(DMPlexSetChart(dm, 0, numVertices));
358     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
359     PetscCall(DMSetLabelValue(dm, "marker", 0, markerLeft));
360     PetscCall(DMSetLabelValue(dm, "marker", 1, markerRight));
361   }
362   PetscCall(DMPlexSymmetrize(dm));
363   PetscCall(DMPlexStratify(dm));
364   /* Build coordinates */
365   PetscCall(DMSetCoordinateDim(dm, cdim));
366   PetscCall(DMGetCoordinateSection(dm, &coordSection));
367   PetscCall(PetscSectionSetNumFields(coordSection, 1));
368   PetscCall(PetscSectionSetChart(coordSection, 0, numVertices));
369   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
370   for (v = 0; v < numVertices; ++v) {
371     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
372     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
373   }
374   PetscCall(PetscSectionSetUp(coordSection));
375   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
376   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
377   PetscCall(PetscObjectSetName((PetscObject) coordinates, "coordinates"));
378   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
379   PetscCall(VecSetBlockSize(coordinates, cdim));
380   PetscCall(VecSetType(coordinates,VECSTANDARD));
381   PetscCall(VecGetArray(coordinates, &coords));
382   coords[0] = lower[0];
383   coords[1] = upper[0];
384   PetscCall(VecRestoreArray(coordinates, &coords));
385   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
386   PetscCall(VecDestroy(&coordinates));
387   PetscFunctionReturn(0);
388 }
389 
390 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
391 {
392   const PetscInt numVertices    = (edges[0]+1)*(edges[1]+1);
393   const PetscInt numEdges       = edges[0]*(edges[1]+1) + (edges[0]+1)*edges[1];
394   PetscInt       markerTop      = 1;
395   PetscInt       markerBottom   = 1;
396   PetscInt       markerRight    = 1;
397   PetscInt       markerLeft     = 1;
398   PetscBool      markerSeparate = PETSC_FALSE;
399   Vec            coordinates;
400   PetscSection   coordSection;
401   PetscScalar    *coords;
402   PetscInt       coordSize;
403   PetscMPIInt    rank;
404   PetscInt       v, vx, vy;
405 
406   PetscFunctionBegin;
407   PetscCall(PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
408   if (markerSeparate) {
409     markerTop    = 3;
410     markerBottom = 1;
411     markerRight  = 2;
412     markerLeft   = 4;
413   }
414   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
415   if (rank == 0) {
416     PetscInt e, ex, ey;
417 
418     PetscCall(DMPlexSetChart(dm, 0, numEdges+numVertices));
419     for (e = 0; e < numEdges; ++e) {
420       PetscCall(DMPlexSetConeSize(dm, e, 2));
421     }
422     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
423     for (vx = 0; vx <= edges[0]; vx++) {
424       for (ey = 0; ey < edges[1]; ey++) {
425         PetscInt edge   = vx*edges[1] + ey + edges[0]*(edges[1]+1);
426         PetscInt vertex = ey*(edges[0]+1) + vx + numEdges;
427         PetscInt cone[2];
428 
429         cone[0] = vertex; cone[1] = vertex+edges[0]+1;
430         PetscCall(DMPlexSetCone(dm, edge, cone));
431         if (vx == edges[0]) {
432           PetscCall(DMSetLabelValue(dm, "marker", edge,    markerRight));
433           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
434           if (ey == edges[1]-1) {
435             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
436             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerRight));
437           }
438         } else if (vx == 0) {
439           PetscCall(DMSetLabelValue(dm, "marker", edge,    markerLeft));
440           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
441           if (ey == edges[1]-1) {
442             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
443             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft));
444           }
445         }
446       }
447     }
448     for (vy = 0; vy <= edges[1]; vy++) {
449       for (ex = 0; ex < edges[0]; ex++) {
450         PetscInt edge   = vy*edges[0]     + ex;
451         PetscInt vertex = vy*(edges[0]+1) + ex + numEdges;
452         PetscInt cone[2];
453 
454         cone[0] = vertex; cone[1] = vertex+1;
455         PetscCall(DMPlexSetCone(dm, edge, cone));
456         if (vy == edges[1]) {
457           PetscCall(DMSetLabelValue(dm, "marker", edge,    markerTop));
458           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
459           if (ex == edges[0]-1) {
460             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
461             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerTop));
462           }
463         } else if (vy == 0) {
464           PetscCall(DMSetLabelValue(dm, "marker", edge,    markerBottom));
465           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
466           if (ex == edges[0]-1) {
467             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
468             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom));
469           }
470         }
471       }
472     }
473   }
474   PetscCall(DMPlexSymmetrize(dm));
475   PetscCall(DMPlexStratify(dm));
476   /* Build coordinates */
477   PetscCall(DMSetCoordinateDim(dm, 2));
478   PetscCall(DMGetCoordinateSection(dm, &coordSection));
479   PetscCall(PetscSectionSetNumFields(coordSection, 1));
480   PetscCall(PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices));
481   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 2));
482   for (v = numEdges; v < numEdges+numVertices; ++v) {
483     PetscCall(PetscSectionSetDof(coordSection, v, 2));
484     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 2));
485   }
486   PetscCall(PetscSectionSetUp(coordSection));
487   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
488   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
489   PetscCall(PetscObjectSetName((PetscObject) coordinates, "coordinates"));
490   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
491   PetscCall(VecSetBlockSize(coordinates, 2));
492   PetscCall(VecSetType(coordinates,VECSTANDARD));
493   PetscCall(VecGetArray(coordinates, &coords));
494   for (vy = 0; vy <= edges[1]; ++vy) {
495     for (vx = 0; vx <= edges[0]; ++vx) {
496       coords[(vy*(edges[0]+1)+vx)*2+0] = lower[0] + ((upper[0] - lower[0])/edges[0])*vx;
497       coords[(vy*(edges[0]+1)+vx)*2+1] = lower[1] + ((upper[1] - lower[1])/edges[1])*vy;
498     }
499   }
500   PetscCall(VecRestoreArray(coordinates, &coords));
501   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
502   PetscCall(VecDestroy(&coordinates));
503   PetscFunctionReturn(0);
504 }
505 
506 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
507 {
508   PetscInt       vertices[3], numVertices;
509   PetscInt       numFaces       = 2*faces[0]*faces[1] + 2*faces[1]*faces[2] + 2*faces[0]*faces[2];
510   PetscInt       markerTop      = 1;
511   PetscInt       markerBottom   = 1;
512   PetscInt       markerFront    = 1;
513   PetscInt       markerBack     = 1;
514   PetscInt       markerRight    = 1;
515   PetscInt       markerLeft     = 1;
516   PetscBool      markerSeparate = PETSC_FALSE;
517   Vec            coordinates;
518   PetscSection   coordSection;
519   PetscScalar    *coords;
520   PetscInt       coordSize;
521   PetscMPIInt    rank;
522   PetscInt       v, vx, vy, vz;
523   PetscInt       voffset, iface=0, cone[4];
524 
525   PetscFunctionBegin;
526   PetscCheck(faces[0] >= 1 && faces[1] >= 1 && faces[2] >= 1,PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
527   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
528   PetscCall(PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
529   if (markerSeparate) {
530     markerBottom = 1;
531     markerTop    = 2;
532     markerFront  = 3;
533     markerBack   = 4;
534     markerRight  = 5;
535     markerLeft   = 6;
536   }
537   vertices[0] = faces[0]+1; vertices[1] = faces[1]+1; vertices[2] = faces[2]+1;
538   numVertices = vertices[0]*vertices[1]*vertices[2];
539   if (rank == 0) {
540     PetscInt f;
541 
542     PetscCall(DMPlexSetChart(dm, 0, numFaces+numVertices));
543     for (f = 0; f < numFaces; ++f) {
544       PetscCall(DMPlexSetConeSize(dm, f, 4));
545     }
546     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
547 
548     /* Side 0 (Top) */
549     for (vy = 0; vy < faces[1]; vy++) {
550       for (vx = 0; vx < faces[0]; vx++) {
551         voffset = numFaces + vertices[0]*vertices[1]*(vertices[2]-1) + vy*vertices[0] + vx;
552         cone[0] = voffset; cone[1] = voffset+1; cone[2] = voffset+vertices[0]+1; cone[3] = voffset+vertices[0];
553         PetscCall(DMPlexSetCone(dm, iface, cone));
554         PetscCall(DMSetLabelValue(dm, "marker", iface, markerTop));
555         PetscCall(DMSetLabelValue(dm, "marker", voffset+0, markerTop));
556         PetscCall(DMSetLabelValue(dm, "marker", voffset+1, markerTop));
557         PetscCall(DMSetLabelValue(dm, "marker", voffset+vertices[0]+0, markerTop));
558         PetscCall(DMSetLabelValue(dm, "marker", voffset+vertices[0]+1, markerTop));
559         iface++;
560       }
561     }
562 
563     /* Side 1 (Bottom) */
564     for (vy = 0; vy < faces[1]; vy++) {
565       for (vx = 0; vx < faces[0]; vx++) {
566         voffset = numFaces + vy*(faces[0]+1) + vx;
567         cone[0] = voffset+1; cone[1] = voffset; cone[2] = voffset+vertices[0]; cone[3] = voffset+vertices[0]+1;
568         PetscCall(DMPlexSetCone(dm, iface, cone));
569         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBottom));
570         PetscCall(DMSetLabelValue(dm, "marker", voffset+0, markerBottom));
571         PetscCall(DMSetLabelValue(dm, "marker", voffset+1, markerBottom));
572         PetscCall(DMSetLabelValue(dm, "marker", voffset+vertices[0]+0, markerBottom));
573         PetscCall(DMSetLabelValue(dm, "marker", voffset+vertices[0]+1, markerBottom));
574         iface++;
575       }
576     }
577 
578     /* Side 2 (Front) */
579     for (vz = 0; vz < faces[2]; vz++) {
580       for (vx = 0; vx < faces[0]; vx++) {
581         voffset = numFaces + vz*vertices[0]*vertices[1] + vx;
582         cone[0] = voffset; cone[1] = voffset+1; cone[2] = voffset+vertices[0]*vertices[1]+1; cone[3] = voffset+vertices[0]*vertices[1];
583         PetscCall(DMPlexSetCone(dm, iface, cone));
584         PetscCall(DMSetLabelValue(dm, "marker", iface, markerFront));
585         PetscCall(DMSetLabelValue(dm, "marker", voffset+0, markerFront));
586         PetscCall(DMSetLabelValue(dm, "marker", voffset+1, markerFront));
587         PetscCall(DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+0, markerFront));
588         PetscCall(DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+1, markerFront));
589         iface++;
590       }
591     }
592 
593     /* Side 3 (Back) */
594     for (vz = 0; vz < faces[2]; vz++) {
595       for (vx = 0; vx < faces[0]; vx++) {
596         voffset = numFaces + vz*vertices[0]*vertices[1] + vertices[0]*(vertices[1]-1) + vx;
597         cone[0] = voffset+vertices[0]*vertices[1]; cone[1] = voffset+vertices[0]*vertices[1]+1;
598         cone[2] = voffset+1; cone[3] = voffset;
599         PetscCall(DMPlexSetCone(dm, iface, cone));
600         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBack));
601         PetscCall(DMSetLabelValue(dm, "marker", voffset+0, markerBack));
602         PetscCall(DMSetLabelValue(dm, "marker", voffset+1, markerBack));
603         PetscCall(DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+0, markerBack));
604         PetscCall(DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+1, markerBack));
605         iface++;
606       }
607     }
608 
609     /* Side 4 (Left) */
610     for (vz = 0; vz < faces[2]; vz++) {
611       for (vy = 0; vy < faces[1]; vy++) {
612         voffset = numFaces + vz*vertices[0]*vertices[1] + vy*vertices[0];
613         cone[0] = voffset; cone[1] = voffset+vertices[0]*vertices[1];
614         cone[2] = voffset+vertices[0]*vertices[1]+vertices[0]; cone[3] = voffset+vertices[0];
615         PetscCall(DMPlexSetCone(dm, iface, cone));
616         PetscCall(DMSetLabelValue(dm, "marker", iface, markerLeft));
617         PetscCall(DMSetLabelValue(dm, "marker", voffset+0, markerLeft));
618         PetscCall(DMSetLabelValue(dm, "marker", voffset+vertices[0]+0, markerLeft));
619         PetscCall(DMSetLabelValue(dm, "marker", voffset+vertices[1]+0, markerLeft));
620         PetscCall(DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+vertices[0], markerLeft));
621         iface++;
622       }
623     }
624 
625     /* Side 5 (Right) */
626     for (vz = 0; vz < faces[2]; vz++) {
627       for (vy = 0; vy < faces[1]; vy++) {
628         voffset = numFaces + vz*vertices[0]*vertices[1] + vy*vertices[0] + faces[0];
629         cone[0] = voffset+vertices[0]*vertices[1]; cone[1] = voffset;
630         cone[2] = voffset+vertices[0]; cone[3] = voffset+vertices[0]*vertices[1]+vertices[0];
631         PetscCall(DMPlexSetCone(dm, iface, cone));
632         PetscCall(DMSetLabelValue(dm, "marker", iface, markerRight));
633         PetscCall(DMSetLabelValue(dm, "marker", voffset+0, markerRight));
634         PetscCall(DMSetLabelValue(dm, "marker", voffset+vertices[0]+0, markerRight));
635         PetscCall(DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+0, markerRight));
636         PetscCall(DMSetLabelValue(dm, "marker", voffset+vertices[0]*vertices[1]+vertices[0], markerRight));
637         iface++;
638       }
639     }
640   }
641   PetscCall(DMPlexSymmetrize(dm));
642   PetscCall(DMPlexStratify(dm));
643   /* Build coordinates */
644   PetscCall(DMSetCoordinateDim(dm, 3));
645   PetscCall(DMGetCoordinateSection(dm, &coordSection));
646   PetscCall(PetscSectionSetNumFields(coordSection, 1));
647   PetscCall(PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices));
648   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 3));
649   for (v = numFaces; v < numFaces+numVertices; ++v) {
650     PetscCall(PetscSectionSetDof(coordSection, v, 3));
651     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 3));
652   }
653   PetscCall(PetscSectionSetUp(coordSection));
654   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
655   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
656   PetscCall(PetscObjectSetName((PetscObject) coordinates, "coordinates"));
657   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
658   PetscCall(VecSetBlockSize(coordinates, 3));
659   PetscCall(VecSetType(coordinates,VECSTANDARD));
660   PetscCall(VecGetArray(coordinates, &coords));
661   for (vz = 0; vz <= faces[2]; ++vz) {
662     for (vy = 0; vy <= faces[1]; ++vy) {
663       for (vx = 0; vx <= faces[0]; ++vx) {
664         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+0] = lower[0] + ((upper[0] - lower[0])/faces[0])*vx;
665         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+1] = lower[1] + ((upper[1] - lower[1])/faces[1])*vy;
666         coords[((vz*(faces[1]+1)+vy)*(faces[0]+1)+vx)*3+2] = lower[2] + ((upper[2] - lower[2])/faces[2])*vz;
667       }
668     }
669   }
670   PetscCall(VecRestoreArray(coordinates, &coords));
671   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
672   PetscCall(VecDestroy(&coordinates));
673   PetscFunctionReturn(0);
674 }
675 
676 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate)
677 {
678   PetscFunctionBegin;
679   PetscValidLogicalCollectiveInt(dm, dim, 2);
680   PetscCall(DMSetDimension(dm, dim-1));
681   PetscCall(DMSetCoordinateDim(dm, dim));
682   switch (dim) {
683     case 1: PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(dm, lower, upper, faces));break;
684     case 2: PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(dm, lower, upper, faces));break;
685     case 3: PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(dm, lower, upper, faces));break;
686     default: SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Dimension not supported: %" PetscInt_FMT, dim);
687   }
688   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
689   PetscFunctionReturn(0);
690 }
691 
692 /*@C
693   DMPlexCreateBoxSurfaceMesh - Creates a mesh on the surface of the tensor product of unit intervals (box) using tensor cells (hexahedra).
694 
695   Collective
696 
697   Input Parameters:
698 + comm        - The communicator for the DM object
699 . dim         - The spatial dimension of the box, so the resulting mesh is has dimension dim-1
700 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
701 . lower       - The lower left corner, or NULL for (0, 0, 0)
702 . upper       - The upper right corner, or NULL for (1, 1, 1)
703 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
704 
705   Output Parameter:
706 . dm  - The DM object
707 
708   Level: beginner
709 
710 .seealso: `DMSetFromOptions()`, `DMPlexCreateBoxMesh()`, `DMPlexCreateFromFile()`, `DMSetType()`, `DMCreate()`
711 @*/
712 PetscErrorCode DMPlexCreateBoxSurfaceMesh(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate, DM *dm)
713 {
714   PetscInt       fac[3] = {1, 1, 1};
715   PetscReal      low[3] = {0, 0, 0};
716   PetscReal      upp[3] = {1, 1, 1};
717 
718   PetscFunctionBegin;
719   PetscCall(DMCreate(comm,dm));
720   PetscCall(DMSetType(*dm,DMPLEX));
721   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(*dm, dim, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, interpolate));
722   PetscFunctionReturn(0);
723 }
724 
725 static PetscErrorCode DMPlexCreateLineMesh_Internal(DM dm,PetscInt segments,PetscReal lower,PetscReal upper,DMBoundaryType bd)
726 {
727   PetscInt       i,fStart,fEnd,numCells = 0,numVerts = 0;
728   PetscInt       numPoints[2],*coneSize,*cones,*coneOrientations;
729   PetscScalar    *vertexCoords;
730   PetscReal      L,maxCell;
731   PetscBool      markerSeparate = PETSC_FALSE;
732   PetscInt       markerLeft  = 1, faceMarkerLeft  = 1;
733   PetscInt       markerRight = 1, faceMarkerRight = 2;
734   PetscBool      wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
735   PetscMPIInt    rank;
736 
737   PetscFunctionBegin;
738   PetscValidPointer(dm,1);
739 
740   PetscCall(DMSetDimension(dm,1));
741   PetscCall(DMCreateLabel(dm,"marker"));
742   PetscCall(DMCreateLabel(dm,"Face Sets"));
743 
744   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject) dm),&rank));
745   if (rank == 0) numCells = segments;
746   if (rank == 0) numVerts = segments + (wrap ? 0 : 1);
747 
748   numPoints[0] = numVerts ; numPoints[1] = numCells;
749   PetscCall(PetscMalloc4(numCells+numVerts,&coneSize,numCells*2,&cones,numCells+numVerts,&coneOrientations,numVerts,&vertexCoords));
750   PetscCall(PetscArrayzero(coneOrientations,numCells+numVerts));
751   for (i = 0; i < numCells; ++i) { coneSize[i] = 2; }
752   for (i = 0; i < numVerts; ++i) { coneSize[numCells+i] = 0; }
753   for (i = 0; i < numCells; ++i) { cones[2*i] = numCells + i%numVerts; cones[2*i+1] = numCells + (i+1)%numVerts; }
754   for (i = 0; i < numVerts; ++i) { vertexCoords[i] = lower + (upper-lower)*((PetscReal)i/(PetscReal)numCells); }
755   PetscCall(DMPlexCreateFromDAG(dm,1,numPoints,coneSize,cones,coneOrientations,vertexCoords));
756   PetscCall(PetscFree4(coneSize,cones,coneOrientations,vertexCoords));
757 
758   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options,((PetscObject)dm)->prefix,"-dm_plex_separate_marker",&markerSeparate,NULL));
759   if (markerSeparate) { markerLeft = faceMarkerLeft; markerRight = faceMarkerRight;}
760   if (!wrap && rank == 0) {
761     PetscCall(DMPlexGetHeightStratum(dm,1,&fStart,&fEnd));
762     PetscCall(DMSetLabelValue(dm,"marker",fStart,markerLeft));
763     PetscCall(DMSetLabelValue(dm,"marker",fEnd-1,markerRight));
764     PetscCall(DMSetLabelValue(dm,"Face Sets",fStart,faceMarkerLeft));
765     PetscCall(DMSetLabelValue(dm,"Face Sets",fEnd-1,faceMarkerRight));
766   }
767   if (wrap) {
768     L       = upper - lower;
769     maxCell = (PetscReal)1.1*(L/(PetscReal)PetscMax(1,segments));
770     PetscCall(DMSetPeriodicity(dm, &maxCell, &lower, &L));
771   }
772   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
773   PetscFunctionReturn(0);
774 }
775 
776 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
777 {
778   DM      boundary, vol;
779   DMLabel bdlabel;
780 
781   PetscFunctionBegin;
782   PetscValidPointer(dm, 1);
783   for (PetscInt i = 0; i < dim; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
784   PetscCall(DMCreate(PetscObjectComm((PetscObject) dm), &boundary));
785   PetscCall(DMSetType(boundary, DMPLEX));
786   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(boundary, dim, faces, lower, upper, PETSC_FALSE));
787   PetscCall(DMPlexGenerate(boundary, NULL, interpolate, &vol));
788   PetscCall(DMGetLabel(vol, "marker", &bdlabel));
789   if (bdlabel) PetscCall(DMPlexLabelComplete(vol, bdlabel));
790   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_FALSE, vol));
791   PetscCall(DMPlexReplace_Static(dm, &vol));
792   PetscCall(DMDestroy(&boundary));
793   PetscFunctionReturn(0);
794 }
795 
796 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
797 {
798   DMLabel        cutLabel = NULL;
799   PetscInt       markerTop      = 1, faceMarkerTop      = 1;
800   PetscInt       markerBottom   = 1, faceMarkerBottom   = 1;
801   PetscInt       markerFront    = 1, faceMarkerFront    = 1;
802   PetscInt       markerBack     = 1, faceMarkerBack     = 1;
803   PetscInt       markerRight    = 1, faceMarkerRight    = 1;
804   PetscInt       markerLeft     = 1, faceMarkerLeft     = 1;
805   PetscInt       dim;
806   PetscBool      markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
807   PetscMPIInt    rank;
808 
809   PetscFunctionBegin;
810   PetscCall(DMGetDimension(dm,&dim));
811   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
812   PetscCall(DMCreateLabel(dm,"marker"));
813   PetscCall(DMCreateLabel(dm,"Face Sets"));
814   PetscCall(PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
815   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ||
816       bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ||
817       bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
818 
819     if (cutMarker) {PetscCall(DMCreateLabel(dm, "periodic_cut")); PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));}
820   }
821   switch (dim) {
822   case 2:
823     faceMarkerTop    = 3;
824     faceMarkerBottom = 1;
825     faceMarkerRight  = 2;
826     faceMarkerLeft   = 4;
827     break;
828   case 3:
829     faceMarkerBottom = 1;
830     faceMarkerTop    = 2;
831     faceMarkerFront  = 3;
832     faceMarkerBack   = 4;
833     faceMarkerRight  = 5;
834     faceMarkerLeft   = 6;
835     break;
836   default:
837     SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Dimension %" PetscInt_FMT " not supported",dim);
838   }
839   PetscCall(PetscOptionsGetBool(((PetscObject) dm)->options,((PetscObject) dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
840   if (markerSeparate) {
841     markerBottom = faceMarkerBottom;
842     markerTop    = faceMarkerTop;
843     markerFront  = faceMarkerFront;
844     markerBack   = faceMarkerBack;
845     markerRight  = faceMarkerRight;
846     markerLeft   = faceMarkerLeft;
847   }
848   {
849     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
850     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
851     const PetscInt numZEdges    = rank == 0 ? edges[2] : 0;
852     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0]+1) : 0;
853     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1]+1) : 0;
854     const PetscInt numZVertices = rank == 0 ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2]+1) : 0;
855     const PetscInt numCells     = numXEdges*numYEdges*numZEdges;
856     const PetscInt numXFaces    = numYEdges*numZEdges;
857     const PetscInt numYFaces    = numXEdges*numZEdges;
858     const PetscInt numZFaces    = numXEdges*numYEdges;
859     const PetscInt numTotXFaces = numXVertices*numXFaces;
860     const PetscInt numTotYFaces = numYVertices*numYFaces;
861     const PetscInt numTotZFaces = numZVertices*numZFaces;
862     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
863     const PetscInt numTotXEdges = numXEdges*numYVertices*numZVertices;
864     const PetscInt numTotYEdges = numYEdges*numXVertices*numZVertices;
865     const PetscInt numTotZEdges = numZEdges*numXVertices*numYVertices;
866     const PetscInt numVertices  = numXVertices*numYVertices*numZVertices;
867     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
868     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
869     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
870     const PetscInt firstYFace   = firstXFace + numTotXFaces;
871     const PetscInt firstZFace   = firstYFace + numTotYFaces;
872     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
873     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
874     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
875     Vec            coordinates;
876     PetscSection   coordSection;
877     PetscScalar   *coords;
878     PetscInt       coordSize;
879     PetscInt       v, vx, vy, vz;
880     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
881 
882     PetscCall(DMPlexSetChart(dm, 0, numCells+numFaces+numEdges+numVertices));
883     for (c = 0; c < numCells; c++) {
884       PetscCall(DMPlexSetConeSize(dm, c, 6));
885     }
886     for (f = firstXFace; f < firstXFace+numFaces; ++f) {
887       PetscCall(DMPlexSetConeSize(dm, f, 4));
888     }
889     for (e = firstXEdge; e < firstXEdge+numEdges; ++e) {
890       PetscCall(DMPlexSetConeSize(dm, e, 2));
891     }
892     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
893     /* Build cells */
894     for (fz = 0; fz < numZEdges; ++fz) {
895       for (fy = 0; fy < numYEdges; ++fy) {
896         for (fx = 0; fx < numXEdges; ++fx) {
897           PetscInt cell    = (fz*numYEdges + fy)*numXEdges + fx;
898           PetscInt faceB   = firstZFace + (fy*numXEdges+fx)*numZVertices +   fz;
899           PetscInt faceT   = firstZFace + (fy*numXEdges+fx)*numZVertices + ((fz+1)%numZVertices);
900           PetscInt faceF   = firstYFace + (fz*numXEdges+fx)*numYVertices +   fy;
901           PetscInt faceK   = firstYFace + (fz*numXEdges+fx)*numYVertices + ((fy+1)%numYVertices);
902           PetscInt faceL   = firstXFace + (fz*numYEdges+fy)*numXVertices +   fx;
903           PetscInt faceR   = firstXFace + (fz*numYEdges+fy)*numXVertices + ((fx+1)%numXVertices);
904                             /* B,  T,  F,  K,  R,  L */
905           PetscInt ornt[6] = {-2,  0,  0, -3,  0, -2}; /* ??? */
906           PetscInt cone[6];
907 
908           /* no boundary twisting in 3D */
909           cone[0] = faceB; cone[1] = faceT; cone[2] = faceF; cone[3] = faceK; cone[4] = faceR; cone[5] = faceL;
910           PetscCall(DMPlexSetCone(dm, cell, cone));
911           PetscCall(DMPlexSetConeOrientation(dm, cell, ornt));
912           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges-1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
913           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges-1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
914           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges-1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
915         }
916       }
917     }
918     /* Build x faces */
919     for (fz = 0; fz < numZEdges; ++fz) {
920       for (fy = 0; fy < numYEdges; ++fy) {
921         for (fx = 0; fx < numXVertices; ++fx) {
922           PetscInt face    = firstXFace + (fz*numYEdges+fy)     *numXVertices+fx;
923           PetscInt edgeL   = firstZEdge + (fy                   *numXVertices+fx)*numZEdges + fz;
924           PetscInt edgeR   = firstZEdge + (((fy+1)%numYVertices)*numXVertices+fx)*numZEdges + fz;
925           PetscInt edgeB   = firstYEdge + (fz                   *numXVertices+fx)*numYEdges + fy;
926           PetscInt edgeT   = firstYEdge + (((fz+1)%numZVertices)*numXVertices+fx)*numYEdges + fy;
927           PetscInt ornt[4] = {0, 0, -1, -1};
928           PetscInt cone[4];
929 
930           if (dim == 3) {
931             /* markers */
932             if (bdX != DM_BOUNDARY_PERIODIC) {
933               if (fx == numXVertices-1) {
934                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight));
935                 PetscCall(DMSetLabelValue(dm, "marker", face, markerRight));
936               }
937               else if (fx == 0) {
938                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft));
939                 PetscCall(DMSetLabelValue(dm, "marker", face, markerLeft));
940               }
941             }
942           }
943           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
944           PetscCall(DMPlexSetCone(dm, face, cone));
945           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
946         }
947       }
948     }
949     /* Build y faces */
950     for (fz = 0; fz < numZEdges; ++fz) {
951       for (fx = 0; fx < numXEdges; ++fx) {
952         for (fy = 0; fy < numYVertices; ++fy) {
953           PetscInt face    = firstYFace + (fz*numXEdges+fx)*numYVertices + fy;
954           PetscInt edgeL   = firstZEdge + (fy*numXVertices+  fx)*numZEdges + fz;
955           PetscInt edgeR   = firstZEdge + (fy*numXVertices+((fx+1)%numXVertices))*numZEdges + fz;
956           PetscInt edgeB   = firstXEdge + (fz                   *numYVertices+fy)*numXEdges + fx;
957           PetscInt edgeT   = firstXEdge + (((fz+1)%numZVertices)*numYVertices+fy)*numXEdges + fx;
958           PetscInt ornt[4] = {0, 0, -1, -1};
959           PetscInt cone[4];
960 
961           if (dim == 3) {
962             /* markers */
963             if (bdY != DM_BOUNDARY_PERIODIC) {
964               if (fy == numYVertices-1) {
965                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack));
966                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBack));
967               }
968               else if (fy == 0) {
969                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront));
970                 PetscCall(DMSetLabelValue(dm, "marker", face, markerFront));
971               }
972             }
973           }
974           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
975           PetscCall(DMPlexSetCone(dm, face, cone));
976           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
977         }
978       }
979     }
980     /* Build z faces */
981     for (fy = 0; fy < numYEdges; ++fy) {
982       for (fx = 0; fx < numXEdges; ++fx) {
983         for (fz = 0; fz < numZVertices; fz++) {
984           PetscInt face    = firstZFace + (fy*numXEdges+fx)*numZVertices + fz;
985           PetscInt edgeL   = firstYEdge + (fz*numXVertices+  fx)*numYEdges + fy;
986           PetscInt edgeR   = firstYEdge + (fz*numXVertices+((fx+1)%numXVertices))*numYEdges + fy;
987           PetscInt edgeB   = firstXEdge + (fz*numYVertices+  fy)*numXEdges + fx;
988           PetscInt edgeT   = firstXEdge + (fz*numYVertices+((fy+1)%numYVertices))*numXEdges + fx;
989           PetscInt ornt[4] = {0, 0, -1, -1};
990           PetscInt cone[4];
991 
992           if (dim == 2) {
993             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges-1) {edgeR += numYEdges-1-2*fy; ornt[1] = -1;}
994             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges-1) {edgeT += numXEdges-1-2*fx; ornt[2] =  0;}
995             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges-1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
996             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges-1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
997           } else {
998             /* markers */
999             if (bdZ != DM_BOUNDARY_PERIODIC) {
1000               if (fz == numZVertices-1) {
1001                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop));
1002                 PetscCall(DMSetLabelValue(dm, "marker", face, markerTop));
1003               }
1004               else if (fz == 0) {
1005                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom));
1006                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBottom));
1007               }
1008             }
1009           }
1010           cone[0] = edgeB; cone[1] = edgeR; cone[2] = edgeT; cone[3] = edgeL;
1011           PetscCall(DMPlexSetCone(dm, face, cone));
1012           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1013         }
1014       }
1015     }
1016     /* Build Z edges*/
1017     for (vy = 0; vy < numYVertices; vy++) {
1018       for (vx = 0; vx < numXVertices; vx++) {
1019         for (ez = 0; ez < numZEdges; ez++) {
1020           const PetscInt edge    = firstZEdge  + (vy*numXVertices+vx)*numZEdges + ez;
1021           const PetscInt vertexB = firstVertex + (ez                   *numYVertices+vy)*numXVertices + vx;
1022           const PetscInt vertexT = firstVertex + (((ez+1)%numZVertices)*numYVertices+vy)*numXVertices + vx;
1023           PetscInt       cone[2];
1024 
1025           cone[0] = vertexB; cone[1] = vertexT;
1026           PetscCall(DMPlexSetCone(dm, edge, cone));
1027           if (dim == 3) {
1028             if (bdX != DM_BOUNDARY_PERIODIC) {
1029               if (vx == numXVertices-1) {
1030                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1031                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1032                 if (ez == numZEdges-1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1033               } else if (vx == 0) {
1034                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1035                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1036                 if (ez == numZEdges-1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1037               }
1038             }
1039             if (bdY != DM_BOUNDARY_PERIODIC) {
1040               if (vy == numYVertices-1) {
1041                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1042                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1043                 if (ez == numZEdges-1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1044               } else if (vy == 0) {
1045                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1046                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1047                 if (ez == numZEdges-1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1048               }
1049             }
1050           }
1051         }
1052       }
1053     }
1054     /* Build Y edges*/
1055     for (vz = 0; vz < numZVertices; vz++) {
1056       for (vx = 0; vx < numXVertices; vx++) {
1057         for (ey = 0; ey < numYEdges; ey++) {
1058           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges-1) ? (numXVertices-vx-1) : (vz*numYVertices+((ey+1)%numYVertices))*numXVertices + vx;
1059           const PetscInt edge    = firstYEdge  + (vz*numXVertices+vx)*numYEdges + ey;
1060           const PetscInt vertexF = firstVertex + (vz*numYVertices+ey)*numXVertices + vx;
1061           const PetscInt vertexK = firstVertex + nextv;
1062           PetscInt       cone[2];
1063 
1064           cone[0] = vertexF; cone[1] = vertexK;
1065           PetscCall(DMPlexSetCone(dm, edge, cone));
1066           if (dim == 2) {
1067             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1068               if (vx == numXVertices-1) {
1069                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight));
1070                 PetscCall(DMSetLabelValue(dm, "marker", edge,    markerRight));
1071                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1072                 if (ey == numYEdges-1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1073               } else if (vx == 0) {
1074                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft));
1075                 PetscCall(DMSetLabelValue(dm, "marker", edge,    markerLeft));
1076                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1077                 if (ey == numYEdges-1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1078               }
1079             } else {
1080               if (vx == 0 && cutLabel) {
1081                 PetscCall(DMLabelSetValue(cutLabel, edge,    1));
1082                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1083                 if (ey == numYEdges-1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1084               }
1085             }
1086           } else {
1087             if (bdX != DM_BOUNDARY_PERIODIC) {
1088               if (vx == numXVertices-1) {
1089                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1090                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1091                 if (ey == numYEdges-1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1092               } else if (vx == 0) {
1093                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1094                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1095                 if (ey == numYEdges-1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1096               }
1097             }
1098             if (bdZ != DM_BOUNDARY_PERIODIC) {
1099               if (vz == numZVertices-1) {
1100                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1101                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1102                 if (ey == numYEdges-1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1103               } else if (vz == 0) {
1104                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1105                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1106                 if (ey == numYEdges-1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1107               }
1108             }
1109           }
1110         }
1111       }
1112     }
1113     /* Build X edges*/
1114     for (vz = 0; vz < numZVertices; vz++) {
1115       for (vy = 0; vy < numYVertices; vy++) {
1116         for (ex = 0; ex < numXEdges; ex++) {
1117           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges-1) ? (numYVertices-vy-1)*numXVertices : (vz*numYVertices+vy)*numXVertices + (ex+1)%numXVertices;
1118           const PetscInt edge    = firstXEdge  + (vz*numYVertices+vy)*numXEdges + ex;
1119           const PetscInt vertexL = firstVertex + (vz*numYVertices+vy)*numXVertices + ex;
1120           const PetscInt vertexR = firstVertex + nextv;
1121           PetscInt       cone[2];
1122 
1123           cone[0] = vertexL; cone[1] = vertexR;
1124           PetscCall(DMPlexSetCone(dm, edge, cone));
1125           if (dim == 2) {
1126             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1127               if (vy == numYVertices-1) {
1128                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop));
1129                 PetscCall(DMSetLabelValue(dm, "marker", edge,    markerTop));
1130                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1131                 if (ex == numXEdges-1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1132               } else if (vy == 0) {
1133                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom));
1134                 PetscCall(DMSetLabelValue(dm, "marker", edge,    markerBottom));
1135                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1136                 if (ex == numXEdges-1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1137               }
1138             } else {
1139               if (vy == 0 && cutLabel) {
1140                 PetscCall(DMLabelSetValue(cutLabel, edge,    1));
1141                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1142                 if (ex == numXEdges-1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1143               }
1144             }
1145           } else {
1146             if (bdY != DM_BOUNDARY_PERIODIC) {
1147               if (vy == numYVertices-1) {
1148                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1149                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1150                 if (ex == numXEdges-1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1151               } else if (vy == 0) {
1152                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1153                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1154                 if (ex == numXEdges-1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1155               }
1156             }
1157             if (bdZ != DM_BOUNDARY_PERIODIC) {
1158               if (vz == numZVertices-1) {
1159                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1160                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1161                 if (ex == numXEdges-1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1162               } else if (vz == 0) {
1163                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1164                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1165                 if (ex == numXEdges-1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1166               }
1167             }
1168           }
1169         }
1170       }
1171     }
1172     PetscCall(DMPlexSymmetrize(dm));
1173     PetscCall(DMPlexStratify(dm));
1174     /* Build coordinates */
1175     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1176     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1177     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1178     PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex+numVertices));
1179     for (v = firstVertex; v < firstVertex+numVertices; ++v) {
1180       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1181       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1182     }
1183     PetscCall(PetscSectionSetUp(coordSection));
1184     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1185     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1186     PetscCall(PetscObjectSetName((PetscObject) coordinates, "coordinates"));
1187     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1188     PetscCall(VecSetBlockSize(coordinates, dim));
1189     PetscCall(VecSetType(coordinates,VECSTANDARD));
1190     PetscCall(VecGetArray(coordinates, &coords));
1191     for (vz = 0; vz < numZVertices; ++vz) {
1192       for (vy = 0; vy < numYVertices; ++vy) {
1193         for (vx = 0; vx < numXVertices; ++vx) {
1194           coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+0] = lower[0] + ((upper[0] - lower[0])/numXEdges)*vx;
1195           coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+1] = lower[1] + ((upper[1] - lower[1])/numYEdges)*vy;
1196           if (dim == 3) {
1197             coords[((vz*numYVertices+vy)*numXVertices+vx)*dim+2] = lower[2] + ((upper[2] - lower[2])/numZEdges)*vz;
1198           }
1199         }
1200       }
1201     }
1202     PetscCall(VecRestoreArray(coordinates, &coords));
1203     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1204     PetscCall(VecDestroy(&coordinates));
1205   }
1206   PetscFunctionReturn(0);
1207 }
1208 
1209 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1210 {
1211   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1212   PetscInt       fac[3] = {0, 0, 0}, d;
1213 
1214   PetscFunctionBegin;
1215   PetscValidPointer(dm, 1);
1216   PetscValidLogicalCollectiveInt(dm, dim, 2);
1217   PetscCall(DMSetDimension(dm, dim));
1218   for (d = 0; d < dim; ++d) {fac[d] = faces[d]; bdt[d] = periodicity[d];}
1219   PetscCall(DMPlexCreateCubeMesh_Internal(dm, lower, upper, fac, bdt[0], bdt[1], bdt[2]));
1220   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST ||
1221       periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST ||
1222       (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
1223     PetscReal L[3]       = {-1., -1., 0.};
1224     PetscReal maxCell[3] = {-1., -1., 0.};
1225 
1226     for (d = 0; d < dim; ++d) {
1227       if (periodicity[d] != DM_BOUNDARY_NONE) {
1228         L[d]       = upper[d] - lower[d];
1229         maxCell[d] = 1.1 * (L[d] / PetscMax(1, faces[d]));
1230       }
1231     }
1232     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1233   }
1234   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
1235   PetscFunctionReturn(0);
1236 }
1237 
1238 static PetscErrorCode DMPlexCreateBoxMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1239 {
1240   PetscFunctionBegin;
1241   if (dim == 1)      PetscCall(DMPlexCreateLineMesh_Internal(dm, faces[0], lower[0], upper[0], periodicity[0]));
1242   else if (simplex)  PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1243   else               PetscCall(DMPlexCreateBoxMesh_Tensor_Internal(dm, dim, faces, lower, upper, periodicity));
1244   if (!interpolate && dim > 1 && !simplex) {
1245     DM udm;
1246 
1247     PetscCall(DMPlexUninterpolate(dm, &udm));
1248     PetscCall(DMPlexCopyCoordinates(dm, udm));
1249     PetscCall(DMPlexReplace_Static(dm, &udm));
1250   }
1251   PetscFunctionReturn(0);
1252 }
1253 
1254 /*@C
1255   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
1256 
1257   Collective
1258 
1259   Input Parameters:
1260 + comm        - The communicator for the DM object
1261 . dim         - The spatial dimension
1262 . simplex     - PETSC_TRUE for simplices, PETSC_FALSE for tensor cells
1263 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1264 . lower       - The lower left corner, or NULL for (0, 0, 0)
1265 . upper       - The upper right corner, or NULL for (1, 1, 1)
1266 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
1267 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1268 
1269   Output Parameter:
1270 . dm  - The DM object
1271 
1272   Note: If you want to customize this mesh using options, you just need to
1273 $  DMCreate(comm, &dm);
1274 $  DMSetType(dm, DMPLEX);
1275 $  DMSetFromOptions(dm);
1276 and use the options on the DMSetFromOptions() page.
1277 
1278   Here is the numbering returned for 2 faces in each direction for tensor cells:
1279 $ 10---17---11---18----12
1280 $  |         |         |
1281 $  |         |         |
1282 $ 20    2   22    3    24
1283 $  |         |         |
1284 $  |         |         |
1285 $  7---15----8---16----9
1286 $  |         |         |
1287 $  |         |         |
1288 $ 19    0   21    1   23
1289 $  |         |         |
1290 $  |         |         |
1291 $  4---13----5---14----6
1292 
1293 and for simplicial cells
1294 
1295 $ 14----8---15----9----16
1296 $  |\     5  |\      7 |
1297 $  | \       | \       |
1298 $ 13   2    14    3    15
1299 $  | 4   \   | 6   \   |
1300 $  |       \ |       \ |
1301 $ 11----6---12----7----13
1302 $  |\        |\        |
1303 $  | \    1  | \     3 |
1304 $ 10   0    11    1    12
1305 $  | 0   \   | 2   \   |
1306 $  |       \ |       \ |
1307 $  8----4----9----5----10
1308 
1309   Level: beginner
1310 
1311 .seealso: `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
1312 @*/
1313 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
1314 {
1315   PetscInt       fac[3] = {1, 1, 1};
1316   PetscReal      low[3] = {0, 0, 0};
1317   PetscReal      upp[3] = {1, 1, 1};
1318   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1319 
1320   PetscFunctionBegin;
1321   PetscCall(DMCreate(comm,dm));
1322   PetscCall(DMSetType(*dm,DMPLEX));
1323   PetscCall(DMPlexCreateBoxMesh_Internal(*dm, dim, simplex, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt, interpolate));
1324   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1325   PetscFunctionReturn(0);
1326 }
1327 
1328 static PetscErrorCode DMPlexCreateWedgeBoxMesh_Internal(DM dm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1329 {
1330   DM             bdm, vol;
1331   PetscInt       i;
1332 
1333   PetscFunctionBegin;
1334   for (i = 0; i < 3; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE,PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Periodicity not yet supported");
1335   PetscCall(DMCreate(PetscObjectComm((PetscObject) dm), &bdm));
1336   PetscCall(DMSetType(bdm, DMPLEX));
1337   PetscCall(DMSetDimension(bdm, 2));
1338   PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(bdm, 2, faces, lower, upper, periodicity, PETSC_TRUE));
1339   PetscCall(DMPlexExtrude(bdm, faces[2], upper[2] - lower[2], PETSC_TRUE, PETSC_FALSE, NULL, NULL, &vol));
1340   PetscCall(DMDestroy(&bdm));
1341   PetscCall(DMPlexReplace_Static(dm, &vol));
1342   if (lower[2] != 0.0) {
1343     Vec          v;
1344     PetscScalar *x;
1345     PetscInt     cDim, n;
1346 
1347     PetscCall(DMGetCoordinatesLocal(dm, &v));
1348     PetscCall(VecGetBlockSize(v, &cDim));
1349     PetscCall(VecGetLocalSize(v, &n));
1350     PetscCall(VecGetArray(v, &x));
1351     x   += cDim;
1352     for (i = 0; i < n; i += cDim) x[i] += lower[2];
1353     PetscCall(VecRestoreArray(v,&x));
1354     PetscCall(DMSetCoordinatesLocal(dm, v));
1355   }
1356   PetscFunctionReturn(0);
1357 }
1358 
1359 /*@
1360   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tesselating the (x,y) plane and extruding in the third direction using wedge cells.
1361 
1362   Collective
1363 
1364   Input Parameters:
1365 + comm        - The communicator for the DM object
1366 . faces       - Number of faces per dimension, or NULL for (1, 1, 1)
1367 . lower       - The lower left corner, or NULL for (0, 0, 0)
1368 . upper       - The upper right corner, or NULL for (1, 1, 1)
1369 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
1370 . orderHeight - If PETSC_TRUE, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1371 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1372 
1373   Output Parameter:
1374 . dm  - The DM object
1375 
1376   Level: beginner
1377 
1378 .seealso: `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateWedgeCylinderMesh()`, `DMExtrude()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1379 @*/
1380 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool orderHeight, PetscBool interpolate, DM *dm)
1381 {
1382   PetscInt       fac[3] = {1, 1, 1};
1383   PetscReal      low[3] = {0, 0, 0};
1384   PetscReal      upp[3] = {1, 1, 1};
1385   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1386 
1387   PetscFunctionBegin;
1388   PetscCall(DMCreate(comm,dm));
1389   PetscCall(DMSetType(*dm,DMPLEX));
1390   PetscCall(DMPlexCreateWedgeBoxMesh_Internal(*dm, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt));
1391   if (!interpolate) {
1392     DM udm;
1393 
1394     PetscCall(DMPlexUninterpolate(*dm, &udm));
1395     PetscCall(DMPlexReplace_Static(*dm, &udm));
1396   }
1397   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1398   PetscFunctionReturn(0);
1399 }
1400 
1401 /*@C
1402   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all DM options in the database.
1403 
1404   Logically Collective on dm
1405 
1406   Input Parameters:
1407 + dm - the DM context
1408 - prefix - the prefix to prepend to all option names
1409 
1410   Notes:
1411   A hyphen (-) must NOT be given at the beginning of the prefix name.
1412   The first character of all runtime options is AUTOMATICALLY the hyphen.
1413 
1414   Level: advanced
1415 
1416 .seealso: `SNESSetFromOptions()`
1417 @*/
1418 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
1419 {
1420   DM_Plex       *mesh = (DM_Plex *) dm->data;
1421 
1422   PetscFunctionBegin;
1423   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1424   PetscCall(PetscObjectSetOptionsPrefix((PetscObject) dm, prefix));
1425   PetscCall(PetscObjectSetOptionsPrefix((PetscObject) mesh->partitioner, prefix));
1426   PetscFunctionReturn(0);
1427 }
1428 
1429 /* Remap geometry to cylinder
1430    TODO: This only works for a single refinement, then it is broken
1431 
1432      Interior square: Linear interpolation is correct
1433      The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
1434      such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
1435 
1436        phi     = arctan(y/x)
1437        d_close = sqrt(1/8 + 1/4 sin^2(phi))
1438        d_far   = sqrt(1/2 + sin^2(phi))
1439 
1440      so we remap them using
1441 
1442        x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
1443        y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
1444 
1445      If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
1446 */
1447 static void snapToCylinder(PetscInt dim, PetscInt Nf, PetscInt NfAux,
1448                            const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
1449                            const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
1450                            PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
1451 {
1452   const PetscReal dis = 1.0/PetscSqrtReal(2.0);
1453   const PetscReal ds2 = 0.5*dis;
1454 
1455   if ((PetscAbsScalar(u[0]) <= ds2) && (PetscAbsScalar(u[1]) <= ds2)) {
1456     f0[0] = u[0];
1457     f0[1] = u[1];
1458   } else {
1459     PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
1460 
1461     x    = PetscRealPart(u[0]);
1462     y    = PetscRealPart(u[1]);
1463     phi  = PetscAtan2Real(y, x);
1464     sinp = PetscSinReal(phi);
1465     cosp = PetscCosReal(phi);
1466     if ((PetscAbsReal(phi) > PETSC_PI/4.0) && (PetscAbsReal(phi) < 3.0*PETSC_PI/4.0)) {
1467       dc = PetscAbsReal(ds2/sinp);
1468       df = PetscAbsReal(dis/sinp);
1469       xc = ds2*x/PetscAbsReal(y);
1470       yc = ds2*PetscSignReal(y);
1471     } else {
1472       dc = PetscAbsReal(ds2/cosp);
1473       df = PetscAbsReal(dis/cosp);
1474       xc = ds2*PetscSignReal(x);
1475       yc = ds2*y/PetscAbsReal(x);
1476     }
1477     f0[0] = xc + (u[0] - xc)*(1.0 - dc)/(df - dc);
1478     f0[1] = yc + (u[1] - yc)*(1.0 - dc)/(df - dc);
1479   }
1480   f0[2] = u[2];
1481 }
1482 
1483 static PetscErrorCode DMPlexCreateHexCylinderMesh_Internal(DM dm, DMBoundaryType periodicZ)
1484 {
1485   const PetscInt dim = 3;
1486   PetscInt       numCells, numVertices;
1487   PetscMPIInt    rank;
1488 
1489   PetscFunctionBegin;
1490   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank));
1491   PetscCall(DMSetDimension(dm, dim));
1492   /* Create topology */
1493   {
1494     PetscInt cone[8], c;
1495 
1496     numCells    = rank == 0 ?  5 : 0;
1497     numVertices = rank == 0 ? 16 : 0;
1498     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1499       numCells   *= 3;
1500       numVertices = rank == 0 ? 24 : 0;
1501     }
1502     PetscCall(DMPlexSetChart(dm, 0, numCells+numVertices));
1503     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 8));
1504     PetscCall(DMSetUp(dm));
1505     if (rank == 0) {
1506       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1507         cone[0] = 15; cone[1] = 18; cone[2] = 17; cone[3] = 16;
1508         cone[4] = 31; cone[5] = 32; cone[6] = 33; cone[7] = 34;
1509         PetscCall(DMPlexSetCone(dm, 0, cone));
1510         cone[0] = 16; cone[1] = 17; cone[2] = 24; cone[3] = 23;
1511         cone[4] = 32; cone[5] = 36; cone[6] = 37; cone[7] = 33; /* 22 25 26 21 */
1512         PetscCall(DMPlexSetCone(dm, 1, cone));
1513         cone[0] = 18; cone[1] = 27; cone[2] = 24; cone[3] = 17;
1514         cone[4] = 34; cone[5] = 33; cone[6] = 37; cone[7] = 38;
1515         PetscCall(DMPlexSetCone(dm, 2, cone));
1516         cone[0] = 29; cone[1] = 27; cone[2] = 18; cone[3] = 15;
1517         cone[4] = 35; cone[5] = 31; cone[6] = 34; cone[7] = 38;
1518         PetscCall(DMPlexSetCone(dm, 3, cone));
1519         cone[0] = 29; cone[1] = 15; cone[2] = 16; cone[3] = 23;
1520         cone[4] = 35; cone[5] = 36; cone[6] = 32; cone[7] = 31;
1521         PetscCall(DMPlexSetCone(dm, 4, cone));
1522 
1523         cone[0] = 31; cone[1] = 34; cone[2] = 33; cone[3] = 32;
1524         cone[4] = 19; cone[5] = 22; cone[6] = 21; cone[7] = 20;
1525         PetscCall(DMPlexSetCone(dm, 5, cone));
1526         cone[0] = 32; cone[1] = 33; cone[2] = 37; cone[3] = 36;
1527         cone[4] = 22; cone[5] = 25; cone[6] = 26; cone[7] = 21;
1528         PetscCall(DMPlexSetCone(dm, 6, cone));
1529         cone[0] = 34; cone[1] = 38; cone[2] = 37; cone[3] = 33;
1530         cone[4] = 20; cone[5] = 21; cone[6] = 26; cone[7] = 28;
1531         PetscCall(DMPlexSetCone(dm, 7, cone));
1532         cone[0] = 35; cone[1] = 38; cone[2] = 34; cone[3] = 31;
1533         cone[4] = 30; cone[5] = 19; cone[6] = 20; cone[7] = 28;
1534         PetscCall(DMPlexSetCone(dm, 8, cone));
1535         cone[0] = 35; cone[1] = 31; cone[2] = 32; cone[3] = 36;
1536         cone[4] = 30; cone[5] = 25; cone[6] = 22; cone[7] = 19;
1537         PetscCall(DMPlexSetCone(dm, 9, cone));
1538 
1539         cone[0] = 19; cone[1] = 20; cone[2] = 21; cone[3] = 22;
1540         cone[4] = 15; cone[5] = 16; cone[6] = 17; cone[7] = 18;
1541         PetscCall(DMPlexSetCone(dm, 10, cone));
1542         cone[0] = 22; cone[1] = 21; cone[2] = 26; cone[3] = 25;
1543         cone[4] = 16; cone[5] = 23; cone[6] = 24; cone[7] = 17;
1544         PetscCall(DMPlexSetCone(dm, 11, cone));
1545         cone[0] = 20; cone[1] = 28; cone[2] = 26; cone[3] = 21;
1546         cone[4] = 18; cone[5] = 17; cone[6] = 24; cone[7] = 27;
1547         PetscCall(DMPlexSetCone(dm, 12, cone));
1548         cone[0] = 30; cone[1] = 28; cone[2] = 20; cone[3] = 19;
1549         cone[4] = 29; cone[5] = 15; cone[6] = 18; cone[7] = 27;
1550         PetscCall(DMPlexSetCone(dm, 13, cone));
1551         cone[0] = 30; cone[1] = 19; cone[2] = 22; cone[3] = 25;
1552         cone[4] = 29; cone[5] = 23; cone[6] = 16; cone[7] = 15;
1553         PetscCall(DMPlexSetCone(dm, 14, cone));
1554       } else {
1555         cone[0] =  5; cone[1] =  8; cone[2] =  7; cone[3] =  6;
1556         cone[4] =  9; cone[5] = 12; cone[6] = 11; cone[7] = 10;
1557         PetscCall(DMPlexSetCone(dm, 0, cone));
1558         cone[0] =  6; cone[1] =  7; cone[2] = 14; cone[3] = 13;
1559         cone[4] = 12; cone[5] = 15; cone[6] = 16; cone[7] = 11;
1560         PetscCall(DMPlexSetCone(dm, 1, cone));
1561         cone[0] =  8; cone[1] = 17; cone[2] = 14; cone[3] =  7;
1562         cone[4] = 10; cone[5] = 11; cone[6] = 16; cone[7] = 18;
1563         PetscCall(DMPlexSetCone(dm, 2, cone));
1564         cone[0] = 19; cone[1] = 17; cone[2] =  8; cone[3] =  5;
1565         cone[4] = 20; cone[5] =  9; cone[6] = 10; cone[7] = 18;
1566         PetscCall(DMPlexSetCone(dm, 3, cone));
1567         cone[0] = 19; cone[1] =  5; cone[2] =  6; cone[3] = 13;
1568         cone[4] = 20; cone[5] = 15; cone[6] = 12; cone[7] =  9;
1569         PetscCall(DMPlexSetCone(dm, 4, cone));
1570       }
1571     }
1572     PetscCall(DMPlexSymmetrize(dm));
1573     PetscCall(DMPlexStratify(dm));
1574   }
1575   /* Create cube geometry */
1576   {
1577     Vec             coordinates;
1578     PetscSection    coordSection;
1579     PetscScalar    *coords;
1580     PetscInt        coordSize, v;
1581     const PetscReal dis = 1.0/PetscSqrtReal(2.0);
1582     const PetscReal ds2 = dis/2.0;
1583 
1584     /* Build coordinates */
1585     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1586     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1587     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1588     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells+numVertices));
1589     for (v = numCells; v < numCells+numVertices; ++v) {
1590       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1591       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1592     }
1593     PetscCall(PetscSectionSetUp(coordSection));
1594     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1595     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1596     PetscCall(PetscObjectSetName((PetscObject) coordinates, "coordinates"));
1597     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1598     PetscCall(VecSetBlockSize(coordinates, dim));
1599     PetscCall(VecSetType(coordinates,VECSTANDARD));
1600     PetscCall(VecGetArray(coordinates, &coords));
1601     if (rank == 0) {
1602       coords[0*dim+0] = -ds2; coords[0*dim+1] = -ds2; coords[0*dim+2] = 0.0;
1603       coords[1*dim+0] =  ds2; coords[1*dim+1] = -ds2; coords[1*dim+2] = 0.0;
1604       coords[2*dim+0] =  ds2; coords[2*dim+1] =  ds2; coords[2*dim+2] = 0.0;
1605       coords[3*dim+0] = -ds2; coords[3*dim+1] =  ds2; coords[3*dim+2] = 0.0;
1606       coords[4*dim+0] = -ds2; coords[4*dim+1] = -ds2; coords[4*dim+2] = 1.0;
1607       coords[5*dim+0] = -ds2; coords[5*dim+1] =  ds2; coords[5*dim+2] = 1.0;
1608       coords[6*dim+0] =  ds2; coords[6*dim+1] =  ds2; coords[6*dim+2] = 1.0;
1609       coords[7*dim+0] =  ds2; coords[7*dim+1] = -ds2; coords[7*dim+2] = 1.0;
1610       coords[ 8*dim+0] =  dis; coords[ 8*dim+1] = -dis; coords[ 8*dim+2] = 0.0;
1611       coords[ 9*dim+0] =  dis; coords[ 9*dim+1] =  dis; coords[ 9*dim+2] = 0.0;
1612       coords[10*dim+0] =  dis; coords[10*dim+1] = -dis; coords[10*dim+2] = 1.0;
1613       coords[11*dim+0] =  dis; coords[11*dim+1] =  dis; coords[11*dim+2] = 1.0;
1614       coords[12*dim+0] = -dis; coords[12*dim+1] =  dis; coords[12*dim+2] = 0.0;
1615       coords[13*dim+0] = -dis; coords[13*dim+1] =  dis; coords[13*dim+2] = 1.0;
1616       coords[14*dim+0] = -dis; coords[14*dim+1] = -dis; coords[14*dim+2] = 0.0;
1617       coords[15*dim+0] = -dis; coords[15*dim+1] = -dis; coords[15*dim+2] = 1.0;
1618       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1619         /* 15 31 19 */ coords[16*dim+0] = -ds2; coords[16*dim+1] = -ds2; coords[16*dim+2] = 0.5;
1620         /* 16 32 22 */ coords[17*dim+0] =  ds2; coords[17*dim+1] = -ds2; coords[17*dim+2] = 0.5;
1621         /* 17 33 21 */ coords[18*dim+0] =  ds2; coords[18*dim+1] =  ds2; coords[18*dim+2] = 0.5;
1622         /* 18 34 20 */ coords[19*dim+0] = -ds2; coords[19*dim+1] =  ds2; coords[19*dim+2] = 0.5;
1623         /* 29 35 30 */ coords[20*dim+0] = -dis; coords[20*dim+1] = -dis; coords[20*dim+2] = 0.5;
1624         /* 23 36 25 */ coords[21*dim+0] =  dis; coords[21*dim+1] = -dis; coords[21*dim+2] = 0.5;
1625         /* 24 37 26 */ coords[22*dim+0] =  dis; coords[22*dim+1] =  dis; coords[22*dim+2] = 0.5;
1626         /* 27 38 28 */ coords[23*dim+0] = -dis; coords[23*dim+1] =  dis; coords[23*dim+2] = 0.5;
1627       }
1628     }
1629     PetscCall(VecRestoreArray(coordinates, &coords));
1630     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1631     PetscCall(VecDestroy(&coordinates));
1632   }
1633   /* Create periodicity */
1634   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1635     PetscReal      L[3]       = {-1., -1., 0.};
1636     PetscReal      maxCell[3] = {-1., -1., 0.};
1637     PetscReal      lower[3]   = {0.0, 0.0, 0.0};
1638     PetscReal      upper[3]   = {1.0, 1.0, 1.5};
1639     PetscInt       numZCells  = 3;
1640 
1641     L[2]       = upper[2] - lower[2];
1642     maxCell[2] = 1.1 * (L[2] / numZCells);
1643     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1644   }
1645   {
1646     DM          cdm;
1647     PetscDS     cds;
1648     PetscScalar c[2] = {1.0, 1.0};
1649 
1650     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToCylinder));
1651     PetscCall(DMGetCoordinateDM(dm, &cdm));
1652     PetscCall(DMGetDS(cdm, &cds));
1653     PetscCall(PetscDSSetConstants(cds, 2, c));
1654   }
1655   /* Wait for coordinate creation before doing in-place modification */
1656   PetscCall(DMPlexInterpolateInPlace_Internal(dm));
1657   PetscFunctionReturn(0);
1658 }
1659 
1660 /*@
1661   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
1662 
1663   Collective
1664 
1665   Input Parameters:
1666 + comm      - The communicator for the DM object
1667 - periodicZ - The boundary type for the Z direction
1668 
1669   Output Parameter:
1670 . dm  - The DM object
1671 
1672   Note:
1673   Here is the output numbering looking from the bottom of the cylinder:
1674 $       17-----14
1675 $        |     |
1676 $        |  2  |
1677 $        |     |
1678 $ 17-----8-----7-----14
1679 $  |     |     |     |
1680 $  |  3  |  0  |  1  |
1681 $  |     |     |     |
1682 $ 19-----5-----6-----13
1683 $        |     |
1684 $        |  4  |
1685 $        |     |
1686 $       19-----13
1687 $
1688 $ and up through the top
1689 $
1690 $       18-----16
1691 $        |     |
1692 $        |  2  |
1693 $        |     |
1694 $ 18----10----11-----16
1695 $  |     |     |     |
1696 $  |  3  |  0  |  1  |
1697 $  |     |     |     |
1698 $ 20-----9----12-----15
1699 $        |     |
1700 $        |  4  |
1701 $        |     |
1702 $       20-----15
1703 
1704   Level: beginner
1705 
1706 .seealso: `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1707 @*/
1708 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, DMBoundaryType periodicZ, DM *dm)
1709 {
1710   PetscFunctionBegin;
1711   PetscValidPointer(dm, 3);
1712   PetscCall(DMCreate(comm, dm));
1713   PetscCall(DMSetType(*dm, DMPLEX));
1714   PetscCall(DMPlexCreateHexCylinderMesh_Internal(*dm, periodicZ));
1715   PetscFunctionReturn(0);
1716 }
1717 
1718 static PetscErrorCode DMPlexCreateWedgeCylinderMesh_Internal(DM dm, PetscInt n, PetscBool interpolate)
1719 {
1720   const PetscInt dim = 3;
1721   PetscInt       numCells, numVertices, v;
1722   PetscMPIInt    rank;
1723 
1724   PetscFunctionBegin;
1725   PetscCheck(n >= 0,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %" PetscInt_FMT " cannot be negative", n);
1726   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank));
1727   PetscCall(DMSetDimension(dm, dim));
1728   /* Must create the celltype label here so that we do not automatically try to compute the types */
1729   PetscCall(DMCreateLabel(dm, "celltype"));
1730   /* Create topology */
1731   {
1732     PetscInt cone[6], c;
1733 
1734     numCells    = rank == 0 ?        n : 0;
1735     numVertices = rank == 0 ?  2*(n+1) : 0;
1736     PetscCall(DMPlexSetChart(dm, 0, numCells+numVertices));
1737     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
1738     PetscCall(DMSetUp(dm));
1739     for (c = 0; c < numCells; c++) {
1740       cone[0] =  c+n*1; cone[1] = (c+1)%n+n*1; cone[2] = 0+3*n;
1741       cone[3] =  c+n*2; cone[4] = (c+1)%n+n*2; cone[5] = 1+3*n;
1742       PetscCall(DMPlexSetCone(dm, c, cone));
1743       PetscCall(DMPlexSetCellType(dm, c, DM_POLYTOPE_TRI_PRISM_TENSOR));
1744     }
1745     PetscCall(DMPlexSymmetrize(dm));
1746     PetscCall(DMPlexStratify(dm));
1747   }
1748   for (v = numCells; v < numCells+numVertices; ++v) {
1749     PetscCall(DMPlexSetCellType(dm, v, DM_POLYTOPE_POINT));
1750   }
1751   /* Create cylinder geometry */
1752   {
1753     Vec          coordinates;
1754     PetscSection coordSection;
1755     PetscScalar *coords;
1756     PetscInt     coordSize, c;
1757 
1758     /* Build coordinates */
1759     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1760     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1761     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1762     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells+numVertices));
1763     for (v = numCells; v < numCells+numVertices; ++v) {
1764       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1765       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1766     }
1767     PetscCall(PetscSectionSetUp(coordSection));
1768     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1769     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1770     PetscCall(PetscObjectSetName((PetscObject) coordinates, "coordinates"));
1771     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1772     PetscCall(VecSetBlockSize(coordinates, dim));
1773     PetscCall(VecSetType(coordinates,VECSTANDARD));
1774     PetscCall(VecGetArray(coordinates, &coords));
1775     for (c = 0; c < numCells; c++) {
1776       coords[(c+0*n)*dim+0] = PetscCosReal(2.0*c*PETSC_PI/n); coords[(c+0*n)*dim+1] = PetscSinReal(2.0*c*PETSC_PI/n); coords[(c+0*n)*dim+2] = 1.0;
1777       coords[(c+1*n)*dim+0] = PetscCosReal(2.0*c*PETSC_PI/n); coords[(c+1*n)*dim+1] = PetscSinReal(2.0*c*PETSC_PI/n); coords[(c+1*n)*dim+2] = 0.0;
1778     }
1779     if (rank == 0) {
1780       coords[(2*n+0)*dim+0] = 0.0; coords[(2*n+0)*dim+1] = 0.0; coords[(2*n+0)*dim+2] = 1.0;
1781       coords[(2*n+1)*dim+0] = 0.0; coords[(2*n+1)*dim+1] = 0.0; coords[(2*n+1)*dim+2] = 0.0;
1782     }
1783     PetscCall(VecRestoreArray(coordinates, &coords));
1784     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1785     PetscCall(VecDestroy(&coordinates));
1786   }
1787   /* Interpolate */
1788   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
1789   PetscFunctionReturn(0);
1790 }
1791 
1792 /*@
1793   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
1794 
1795   Collective
1796 
1797   Input Parameters:
1798 + comm - The communicator for the DM object
1799 . n    - The number of wedges around the origin
1800 - interpolate - Create edges and faces
1801 
1802   Output Parameter:
1803 . dm  - The DM object
1804 
1805   Level: beginner
1806 
1807 .seealso: `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1808 @*/
1809 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
1810 {
1811   PetscFunctionBegin;
1812   PetscValidPointer(dm, 4);
1813   PetscCall(DMCreate(comm, dm));
1814   PetscCall(DMSetType(*dm, DMPLEX));
1815   PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(*dm, n, interpolate));
1816   PetscFunctionReturn(0);
1817 }
1818 
1819 static inline PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
1820 {
1821   PetscReal prod = 0.0;
1822   PetscInt  i;
1823   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
1824   return PetscSqrtReal(prod);
1825 }
1826 static inline PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
1827 {
1828   PetscReal prod = 0.0;
1829   PetscInt  i;
1830   for (i = 0; i < dim; ++i) prod += x[i]*y[i];
1831   return prod;
1832 }
1833 
1834 /* The first constant is the sphere radius */
1835 static void snapToSphere(PetscInt dim, PetscInt Nf, PetscInt NfAux,
1836                          const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[],
1837                          const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[],
1838                          PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
1839 {
1840   PetscReal r = PetscRealPart(constants[0]);
1841   PetscReal norm2 = 0.0, fac;
1842   PetscInt  n = uOff[1] - uOff[0], d;
1843 
1844   for (d = 0; d < n; ++d) norm2 += PetscSqr(PetscRealPart(u[d]));
1845   fac = r/PetscSqrtReal(norm2);
1846   for (d = 0; d < n; ++d) f0[d] = u[d]*fac;
1847 }
1848 
1849 static PetscErrorCode DMPlexCreateSphereMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, PetscReal R)
1850 {
1851   const PetscInt  embedDim = dim+1;
1852   PetscSection    coordSection;
1853   Vec             coordinates;
1854   PetscScalar    *coords;
1855   PetscReal      *coordsIn;
1856   PetscInt        numCells, numEdges, numVerts, firstVertex, v, firstEdge, coordSize, d, c, e;
1857   PetscMPIInt     rank;
1858 
1859   PetscFunctionBegin;
1860   PetscValidLogicalCollectiveBool(dm, simplex, 3);
1861   PetscCall(DMSetDimension(dm, dim));
1862   PetscCall(DMSetCoordinateDim(dm, dim+1));
1863   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank));
1864   switch (dim) {
1865   case 2:
1866     if (simplex) {
1867       const PetscReal radius    = PetscSqrtReal(1 + PETSC_PHI*PETSC_PHI)/(1.0 + PETSC_PHI);
1868       const PetscReal edgeLen   = 2.0/(1.0 + PETSC_PHI) * (R/radius);
1869       const PetscInt  degree    = 5;
1870       PetscReal       vertex[3] = {0.0, 1.0/(1.0 + PETSC_PHI), PETSC_PHI/(1.0 + PETSC_PHI)};
1871       PetscInt        s[3]      = {1, 1, 1};
1872       PetscInt        cone[3];
1873       PetscInt       *graph, p, i, j, k;
1874 
1875       vertex[0] *= R/radius; vertex[1] *= R/radius; vertex[2] *= R/radius;
1876       numCells    = rank == 0 ? 20 : 0;
1877       numVerts    = rank == 0 ? 12 : 0;
1878       firstVertex = numCells;
1879       /* Use icosahedron, which for a R-sphere has coordinates which are all cyclic permutations of
1880 
1881            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
1882 
1883          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
1884          length is then given by 2/(1+\phi) = 2 * 0.38197 = 0.76393.
1885       */
1886       /* Construct vertices */
1887       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
1888       if (rank == 0) {
1889         for (p = 0, i = 0; p < embedDim; ++p) {
1890           for (s[1] = -1; s[1] < 2; s[1] += 2) {
1891             for (s[2] = -1; s[2] < 2; s[2] += 2) {
1892               for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[(d+p)%embedDim]*vertex[(d+p)%embedDim];
1893               ++i;
1894             }
1895           }
1896         }
1897       }
1898       /* Construct graph */
1899       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
1900       for (i = 0; i < numVerts; ++i) {
1901         for (j = 0, k = 0; j < numVerts; ++j) {
1902           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i*embedDim], &coordsIn[j*embedDim]) - edgeLen) < PETSC_SMALL) {graph[i*numVerts+j] = 1; ++k;}
1903         }
1904         PetscCheck(k == degree,PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid icosahedron, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
1905       }
1906       /* Build Topology */
1907       PetscCall(DMPlexSetChart(dm, 0, numCells+numVerts));
1908       for (c = 0; c < numCells; c++) {
1909         PetscCall(DMPlexSetConeSize(dm, c, embedDim));
1910       }
1911       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
1912       /* Cells */
1913       for (i = 0, c = 0; i < numVerts; ++i) {
1914         for (j = 0; j < i; ++j) {
1915           for (k = 0; k < j; ++k) {
1916             if (graph[i*numVerts+j] && graph[j*numVerts+k] && graph[k*numVerts+i]) {
1917               cone[0] = firstVertex+i; cone[1] = firstVertex+j; cone[2] = firstVertex+k;
1918               /* Check orientation */
1919               {
1920                 const PetscInt epsilon[3][3][3] = {{{0, 0, 0}, {0, 0, 1}, {0, -1, 0}}, {{0, 0, -1}, {0, 0, 0}, {1, 0, 0}}, {{0, 1, 0}, {-1, 0, 0}, {0, 0, 0}}};
1921                 PetscReal normal[3];
1922                 PetscInt  e, f;
1923 
1924                 for (d = 0; d < embedDim; ++d) {
1925                   normal[d] = 0.0;
1926                   for (e = 0; e < embedDim; ++e) {
1927                     for (f = 0; f < embedDim; ++f) {
1928                       normal[d] += epsilon[d][e][f]*(coordsIn[j*embedDim+e] - coordsIn[i*embedDim+e])*(coordsIn[k*embedDim+f] - coordsIn[i*embedDim+f]);
1929                     }
1930                   }
1931                 }
1932                 if (DotReal(embedDim, normal, &coordsIn[i*embedDim]) < 0) {PetscInt tmp = cone[1]; cone[1] = cone[2]; cone[2] = tmp;}
1933               }
1934               PetscCall(DMPlexSetCone(dm, c++, cone));
1935             }
1936           }
1937         }
1938       }
1939       PetscCall(DMPlexSymmetrize(dm));
1940       PetscCall(DMPlexStratify(dm));
1941       PetscCall(PetscFree(graph));
1942     } else {
1943       /*
1944         12-21--13
1945          |     |
1946         25  4  24
1947          |     |
1948   12-25--9-16--8-24--13
1949    |     |     |     |
1950   23  5 17  0 15  3  22
1951    |     |     |     |
1952   10-20--6-14--7-19--11
1953          |     |
1954         20  1  19
1955          |     |
1956         10-18--11
1957          |     |
1958         23  2  22
1959          |     |
1960         12-21--13
1961        */
1962       PetscInt cone[4], ornt[4];
1963 
1964       numCells    = rank == 0 ?  6 : 0;
1965       numEdges    = rank == 0 ? 12 : 0;
1966       numVerts    = rank == 0 ?  8 : 0;
1967       firstVertex = numCells;
1968       firstEdge   = numCells + numVerts;
1969       /* Build Topology */
1970       PetscCall(DMPlexSetChart(dm, 0, numCells+numEdges+numVerts));
1971       for (c = 0; c < numCells; c++) {
1972         PetscCall(DMPlexSetConeSize(dm, c, 4));
1973       }
1974       for (e = firstEdge; e < firstEdge+numEdges; ++e) {
1975         PetscCall(DMPlexSetConeSize(dm, e, 2));
1976       }
1977       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
1978       if (rank == 0) {
1979         /* Cell 0 */
1980         cone[0] = 14; cone[1] = 15; cone[2] = 16; cone[3] = 17;
1981         PetscCall(DMPlexSetCone(dm, 0, cone));
1982         ornt[0] = 0; ornt[1] = 0; ornt[2] = 0; ornt[3] = 0;
1983         PetscCall(DMPlexSetConeOrientation(dm, 0, ornt));
1984         /* Cell 1 */
1985         cone[0] = 18; cone[1] = 19; cone[2] = 14; cone[3] = 20;
1986         PetscCall(DMPlexSetCone(dm, 1, cone));
1987         ornt[0] = 0; ornt[1] = 0; ornt[2] = -1; ornt[3] = 0;
1988         PetscCall(DMPlexSetConeOrientation(dm, 1, ornt));
1989         /* Cell 2 */
1990         cone[0] = 21; cone[1] = 22; cone[2] = 18; cone[3] = 23;
1991         PetscCall(DMPlexSetCone(dm, 2, cone));
1992         ornt[0] = 0; ornt[1] = 0; ornt[2] = -1; ornt[3] = 0;
1993         PetscCall(DMPlexSetConeOrientation(dm, 2, ornt));
1994         /* Cell 3 */
1995         cone[0] = 19; cone[1] = 22; cone[2] = 24; cone[3] = 15;
1996         PetscCall(DMPlexSetCone(dm, 3, cone));
1997         ornt[0] = -1; ornt[1] = -1; ornt[2] = 0; ornt[3] = -1;
1998         PetscCall(DMPlexSetConeOrientation(dm, 3, ornt));
1999         /* Cell 4 */
2000         cone[0] = 16; cone[1] = 24; cone[2] = 21; cone[3] = 25;
2001         PetscCall(DMPlexSetCone(dm, 4, cone));
2002         ornt[0] = -1; ornt[1] = -1; ornt[2] = -1; ornt[3] = 0;
2003         PetscCall(DMPlexSetConeOrientation(dm, 4, ornt));
2004         /* Cell 5 */
2005         cone[0] = 20; cone[1] = 17; cone[2] = 25; cone[3] = 23;
2006         PetscCall(DMPlexSetCone(dm, 5, cone));
2007         ornt[0] = -1; ornt[1] = -1; ornt[2] = -1; ornt[3] = -1;
2008         PetscCall(DMPlexSetConeOrientation(dm, 5, ornt));
2009         /* Edges */
2010         cone[0] =  6; cone[1] =  7;
2011         PetscCall(DMPlexSetCone(dm, 14, cone));
2012         cone[0] =  7; cone[1] =  8;
2013         PetscCall(DMPlexSetCone(dm, 15, cone));
2014         cone[0] =  8; cone[1] =  9;
2015         PetscCall(DMPlexSetCone(dm, 16, cone));
2016         cone[0] =  9; cone[1] =  6;
2017         PetscCall(DMPlexSetCone(dm, 17, cone));
2018         cone[0] = 10; cone[1] = 11;
2019         PetscCall(DMPlexSetCone(dm, 18, cone));
2020         cone[0] = 11; cone[1] =  7;
2021         PetscCall(DMPlexSetCone(dm, 19, cone));
2022         cone[0] =  6; cone[1] = 10;
2023         PetscCall(DMPlexSetCone(dm, 20, cone));
2024         cone[0] = 12; cone[1] = 13;
2025         PetscCall(DMPlexSetCone(dm, 21, cone));
2026         cone[0] = 13; cone[1] = 11;
2027         PetscCall(DMPlexSetCone(dm, 22, cone));
2028         cone[0] = 10; cone[1] = 12;
2029         PetscCall(DMPlexSetCone(dm, 23, cone));
2030         cone[0] = 13; cone[1] =  8;
2031         PetscCall(DMPlexSetCone(dm, 24, cone));
2032         cone[0] = 12; cone[1] =  9;
2033         PetscCall(DMPlexSetCone(dm, 25, cone));
2034       }
2035       PetscCall(DMPlexSymmetrize(dm));
2036       PetscCall(DMPlexStratify(dm));
2037       /* Build coordinates */
2038       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2039       if (rank == 0) {
2040         coordsIn[0*embedDim+0] = -R; coordsIn[0*embedDim+1] =  R; coordsIn[0*embedDim+2] = -R;
2041         coordsIn[1*embedDim+0] =  R; coordsIn[1*embedDim+1] =  R; coordsIn[1*embedDim+2] = -R;
2042         coordsIn[2*embedDim+0] =  R; coordsIn[2*embedDim+1] = -R; coordsIn[2*embedDim+2] = -R;
2043         coordsIn[3*embedDim+0] = -R; coordsIn[3*embedDim+1] = -R; coordsIn[3*embedDim+2] = -R;
2044         coordsIn[4*embedDim+0] = -R; coordsIn[4*embedDim+1] =  R; coordsIn[4*embedDim+2] =  R;
2045         coordsIn[5*embedDim+0] =  R; coordsIn[5*embedDim+1] =  R; coordsIn[5*embedDim+2] =  R;
2046         coordsIn[6*embedDim+0] = -R; coordsIn[6*embedDim+1] = -R; coordsIn[6*embedDim+2] =  R;
2047         coordsIn[7*embedDim+0] =  R; coordsIn[7*embedDim+1] = -R; coordsIn[7*embedDim+2] =  R;
2048       }
2049     }
2050     break;
2051   case 3:
2052     if (simplex) {
2053       const PetscReal edgeLen         = 1.0/PETSC_PHI;
2054       PetscReal       vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
2055       PetscReal       vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
2056       PetscReal       vertexC[4]      = {0.5, 0.5*PETSC_PHI, 0.5/PETSC_PHI, 0.0};
2057       const PetscInt  degree          = 12;
2058       PetscInt        s[4]            = {1, 1, 1};
2059       PetscInt        evenPerm[12][4] = {{0, 1, 2, 3}, {0, 2, 3, 1}, {0, 3, 1, 2}, {1, 0, 3, 2}, {1, 2, 0, 3}, {1, 3, 2, 0},
2060                                          {2, 0, 1, 3}, {2, 1, 3, 0}, {2, 3, 0, 1}, {3, 0, 2, 1}, {3, 1, 0, 2}, {3, 2, 1, 0}};
2061       PetscInt        cone[4];
2062       PetscInt       *graph, p, i, j, k, l;
2063 
2064       vertexA[0] *= R; vertexA[1] *= R; vertexA[2] *= R; vertexA[3] *= R;
2065       vertexB[0] *= R; vertexB[1] *= R; vertexB[2] *= R; vertexB[3] *= R;
2066       vertexC[0] *= R; vertexC[1] *= R; vertexC[2] *= R; vertexC[3] *= R;
2067       numCells    = rank == 0 ? 600 : 0;
2068       numVerts    = rank == 0 ? 120 : 0;
2069       firstVertex = numCells;
2070       /* Use the 600-cell, which for a unit sphere has coordinates which are
2071 
2072            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
2073                (\pm 1,    0,       0,      0)  all cyclic permutations   8
2074            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
2075 
2076          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2077          length is then given by 1/\phi = 0.61803.
2078 
2079          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
2080          http://mathworld.wolfram.com/600-Cell.html
2081       */
2082       /* Construct vertices */
2083       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2084       i    = 0;
2085       if (rank == 0) {
2086         for (s[0] = -1; s[0] < 2; s[0] += 2) {
2087           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2088             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2089               for (s[3] = -1; s[3] < 2; s[3] += 2) {
2090                 for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[d]*vertexA[d];
2091                 ++i;
2092               }
2093             }
2094           }
2095         }
2096         for (p = 0; p < embedDim; ++p) {
2097           s[1] = s[2] = s[3] = 1;
2098           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2099             for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[(d+p)%embedDim]*vertexB[(d+p)%embedDim];
2100             ++i;
2101           }
2102         }
2103         for (p = 0; p < 12; ++p) {
2104           s[3] = 1;
2105           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2106             for (s[1] = -1; s[1] < 2; s[1] += 2) {
2107               for (s[2] = -1; s[2] < 2; s[2] += 2) {
2108                 for (d = 0; d < embedDim; ++d) coordsIn[i*embedDim+d] = s[evenPerm[p][d]]*vertexC[evenPerm[p][d]];
2109                 ++i;
2110               }
2111             }
2112           }
2113         }
2114       }
2115       PetscCheck(i == numVerts,PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertices %" PetscInt_FMT " != %" PetscInt_FMT, i, numVerts);
2116       /* Construct graph */
2117       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2118       for (i = 0; i < numVerts; ++i) {
2119         for (j = 0, k = 0; j < numVerts; ++j) {
2120           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i*embedDim], &coordsIn[j*embedDim]) - edgeLen) < PETSC_SMALL) {graph[i*numVerts+j] = 1; ++k;}
2121         }
2122         PetscCheck(k == degree,PetscObjectComm((PetscObject) dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2123       }
2124       /* Build Topology */
2125       PetscCall(DMPlexSetChart(dm, 0, numCells+numVerts));
2126       for (c = 0; c < numCells; c++) {
2127         PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2128       }
2129       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2130       /* Cells */
2131       if (rank == 0) {
2132         for (i = 0, c = 0; i < numVerts; ++i) {
2133           for (j = 0; j < i; ++j) {
2134             for (k = 0; k < j; ++k) {
2135               for (l = 0; l < k; ++l) {
2136                 if (graph[i*numVerts+j] && graph[j*numVerts+k] && graph[k*numVerts+i] &&
2137                     graph[l*numVerts+i] && graph[l*numVerts+j] && graph[l*numVerts+k]) {
2138                   cone[0] = firstVertex+i; cone[1] = firstVertex+j; cone[2] = firstVertex+k; cone[3] = firstVertex+l;
2139                   /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
2140                   {
2141                     const PetscInt epsilon[4][4][4][4] = {{{{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2142                                                            {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  1}, { 0,  0, -1, 0}},
2143                                                            {{0,  0,  0,  0}, { 0, 0,  0, -1}, { 0,  0, 0,  0}, { 0,  1,  0, 0}},
2144                                                            {{0,  0,  0,  0}, { 0, 0,  1,  0}, { 0, -1, 0,  0}, { 0,  0,  0, 0}}},
2145 
2146                                                           {{{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0, -1}, { 0,  0,  1, 0}},
2147                                                            {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2148                                                            {{0,  0,  0,  1}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, {-1,  0,  0, 0}},
2149                                                            {{0,  0, -1,  0}, { 0, 0,  0,  0}, { 1,  0, 0,  0}, { 0,  0,  0, 0}}},
2150 
2151                                                           {{{0,  0,  0,  0}, { 0, 0,  0,  1}, { 0,  0, 0,  0}, { 0, -1,  0, 0}},
2152                                                            {{0,  0,  0, -1}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 1,  0,  0, 0}},
2153                                                            {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2154                                                            {{0,  1,  0,  0}, {-1, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}}},
2155 
2156                                                           {{{0,  0,  0,  0}, { 0, 0, -1,  0}, { 0,  1, 0,  0}, { 0,  0,  0, 0}},
2157                                                            {{0,  0,  1,  0}, { 0, 0,  0,  0}, {-1,  0, 0,  0}, { 0,  0,  0, 0}},
2158                                                            {{0, -1,  0,  0}, { 1, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}},
2159                                                            {{0,  0,  0,  0}, { 0, 0,  0,  0}, { 0,  0, 0,  0}, { 0,  0,  0, 0}}}};
2160                     PetscReal normal[4];
2161                     PetscInt  e, f, g;
2162 
2163                     for (d = 0; d < embedDim; ++d) {
2164                       normal[d] = 0.0;
2165                       for (e = 0; e < embedDim; ++e) {
2166                         for (f = 0; f < embedDim; ++f) {
2167                           for (g = 0; g < embedDim; ++g) {
2168                             normal[d] += epsilon[d][e][f][g]*(coordsIn[j*embedDim+e] - coordsIn[i*embedDim+e])*(coordsIn[k*embedDim+f] - coordsIn[i*embedDim+f])*(coordsIn[l*embedDim+f] - coordsIn[i*embedDim+f]);
2169                           }
2170                         }
2171                       }
2172                     }
2173                     if (DotReal(embedDim, normal, &coordsIn[i*embedDim]) < 0) {PetscInt tmp = cone[1]; cone[1] = cone[2]; cone[2] = tmp;}
2174                   }
2175                   PetscCall(DMPlexSetCone(dm, c++, cone));
2176                 }
2177               }
2178             }
2179           }
2180         }
2181       }
2182       PetscCall(DMPlexSymmetrize(dm));
2183       PetscCall(DMPlexStratify(dm));
2184       PetscCall(PetscFree(graph));
2185       break;
2186     }
2187   default: SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Unsupported dimension for sphere: %" PetscInt_FMT, dim);
2188   }
2189   /* Create coordinates */
2190   PetscCall(DMGetCoordinateSection(dm, &coordSection));
2191   PetscCall(PetscSectionSetNumFields(coordSection, 1));
2192   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, embedDim));
2193   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex+numVerts));
2194   for (v = firstVertex; v < firstVertex+numVerts; ++v) {
2195     PetscCall(PetscSectionSetDof(coordSection, v, embedDim));
2196     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, embedDim));
2197   }
2198   PetscCall(PetscSectionSetUp(coordSection));
2199   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2200   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2201   PetscCall(VecSetBlockSize(coordinates, embedDim));
2202   PetscCall(PetscObjectSetName((PetscObject) coordinates, "coordinates"));
2203   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2204   PetscCall(VecSetType(coordinates,VECSTANDARD));
2205   PetscCall(VecGetArray(coordinates, &coords));
2206   for (v = 0; v < numVerts; ++v) for (d = 0; d < embedDim; ++d) {coords[v*embedDim+d] = coordsIn[v*embedDim+d];}
2207   PetscCall(VecRestoreArray(coordinates, &coords));
2208   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2209   PetscCall(VecDestroy(&coordinates));
2210   PetscCall(PetscFree(coordsIn));
2211   {
2212     DM          cdm;
2213     PetscDS     cds;
2214     PetscScalar c = R;
2215 
2216     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToSphere));
2217     PetscCall(DMGetCoordinateDM(dm, &cdm));
2218     PetscCall(DMGetDS(cdm, &cds));
2219     PetscCall(PetscDSSetConstants(cds, 1, &c));
2220   }
2221   /* Wait for coordinate creation before doing in-place modification */
2222   if (simplex) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2223   PetscFunctionReturn(0);
2224 }
2225 
2226 typedef void (*TPSEvaluateFunc)(const PetscReal[], PetscReal*, PetscReal[], PetscReal(*)[3]);
2227 
2228 /*
2229  The Schwarz P implicit surface is
2230 
2231      f(x) = cos(x0) + cos(x1) + cos(x2) = 0
2232 */
2233 static void TPSEvaluate_SchwarzP(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2234 {
2235   PetscReal c[3] = {PetscCosReal(y[0] * PETSC_PI), PetscCosReal(y[1] * PETSC_PI), PetscCosReal(y[2] * PETSC_PI)};
2236   PetscReal g[3] = {-PetscSinReal(y[0] * PETSC_PI), -PetscSinReal(y[1] * PETSC_PI), -PetscSinReal(y[2] * PETSC_PI)};
2237   f[0] = c[0] + c[1] + c[2];
2238   for (PetscInt i=0; i<3; i++) {
2239     grad[i] = PETSC_PI * g[i];
2240     for (PetscInt j=0; j<3; j++) {
2241       hess[i][j] = (i == j) ? -PetscSqr(PETSC_PI) * c[i] : 0.;
2242     }
2243   }
2244 }
2245 
2246 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2247 static PetscErrorCode TPSExtrudeNormalFunc_SchwarzP(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx) {
2248   for (PetscInt i=0; i<3; i++) {
2249     u[i] = -PETSC_PI * PetscSinReal(x[i] * PETSC_PI);
2250   }
2251   return 0;
2252 }
2253 
2254 /*
2255  The Gyroid implicit surface is
2256 
2257  f(x,y,z) = sin(pi * x) * cos (pi * (y + 1/2))  + sin(pi * (y + 1/2)) * cos(pi * (z + 1/4)) + sin(pi * (z + 1/4)) * cos(pi * x)
2258 
2259 */
2260 static void TPSEvaluate_Gyroid(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2261 {
2262   PetscReal s[3] = {PetscSinReal(PETSC_PI * y[0]), PetscSinReal(PETSC_PI * (y[1] + .5)), PetscSinReal(PETSC_PI * (y[2] + .25))};
2263   PetscReal c[3] = {PetscCosReal(PETSC_PI * y[0]), PetscCosReal(PETSC_PI * (y[1] + .5)), PetscCosReal(PETSC_PI * (y[2] + .25))};
2264   f[0] = s[0] * c[1] + s[1] * c[2] + s[2] * c[0];
2265   grad[0] = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2266   grad[1] = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2267   grad[2] = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2268   hess[0][0] = -PetscSqr(PETSC_PI) * (s[0] * c[1] + s[2] * c[0]);
2269   hess[0][1] = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2270   hess[0][2] = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2271   hess[1][0] = -PetscSqr(PETSC_PI) * (s[1] * c[2] + s[0] * c[1]);
2272   hess[1][1] = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2273   hess[2][2] = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2274   hess[2][0] = -PetscSqr(PETSC_PI) * (s[2] * c[0] + s[1] * c[2]);
2275   hess[2][1] = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2276   hess[2][2] = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2277 }
2278 
2279 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2280 static PetscErrorCode TPSExtrudeNormalFunc_Gyroid(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx) {
2281   PetscReal s[3] = {PetscSinReal(PETSC_PI * x[0]), PetscSinReal(PETSC_PI * (x[1] + .5)), PetscSinReal(PETSC_PI * (x[2] + .25))};
2282   PetscReal c[3] = {PetscCosReal(PETSC_PI * x[0]), PetscCosReal(PETSC_PI * (x[1] + .5)), PetscCosReal(PETSC_PI * (x[2] + .25))};
2283   u[0] = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2284   u[1] = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2285   u[2] = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2286   return 0;
2287 }
2288 
2289 /*
2290    We wish to solve
2291 
2292          min_y || y - x ||^2  subject to f(y) = 0
2293 
2294    Let g(y) = grad(f).  The minimization problem is equivalent to asking to satisfy
2295    f(y) = 0 and (y-x) is parallel to g(y).  We do this by using Householder QR to obtain a basis for the
2296    tangent space and ask for both components in the tangent space to be zero.
2297 
2298    Take g to be a column vector and compute the "full QR" factorization Q R = g,
2299    where Q = I - 2 n n^T is a symmetric orthogonal matrix.
2300    The first column of Q is parallel to g so the remaining two columns span the null space.
2301    Let Qn = Q[:,1:] be those remaining columns.  Then Qn Qn^T is an orthogonal projector into the tangent space.
2302    Since Q is symmetric, this is equivalent to multipyling by Q and taking the last two entries.
2303    In total, we have a system of 3 equations in 3 unknowns:
2304 
2305      f(y) = 0                       1 equation
2306      Qn^T (y - x) = 0               2 equations
2307 
2308    Here, we compute the residual and Jacobian of this system.
2309 */
2310 static void TPSNearestPointResJac(TPSEvaluateFunc feval, const PetscScalar x[], const PetscScalar y[], PetscScalar res[], PetscScalar J[])
2311 {
2312   PetscReal yreal[3] = {PetscRealPart(y[0]), PetscRealPart(y[1]), PetscRealPart(y[2])};
2313   PetscReal d[3] = {PetscRealPart(y[0] - x[0]), PetscRealPart(y[1] - x[1]), PetscRealPart(y[2] - x[2])};
2314   PetscReal f, grad[3], n[3], norm, norm_y[3], nd, nd_y[3], sign;
2315   PetscReal n_y[3][3] = {{0,0,0},{0,0,0},{0,0,0}};
2316 
2317   feval(yreal, &f, grad, n_y);
2318 
2319   for (PetscInt i=0; i<3; i++) n[i] = grad[i];
2320   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2321   for (PetscInt i=0; i<3; i++) {
2322     norm_y[i] = 1. / norm * n[i] * n_y[i][i];
2323   }
2324 
2325   // Define the Householder reflector
2326   sign = n[0] >= 0 ? 1. : -1.;
2327   n[0] += norm * sign;
2328   for (PetscInt i=0; i<3; i++) n_y[0][i] += norm_y[i] * sign;
2329 
2330   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2331   norm_y[0] = 1. / norm * (n[0] * n_y[0][0]);
2332   norm_y[1] = 1. / norm * (n[0] * n_y[0][1] + n[1] * n_y[1][1]);
2333   norm_y[2] = 1. / norm * (n[0] * n_y[0][2] + n[2] * n_y[2][2]);
2334 
2335   for (PetscInt i=0; i<3; i++) {
2336     n[i] /= norm;
2337     for (PetscInt j=0; j<3; j++) {
2338       // note that n[i] is n_old[i]/norm when executing the code below
2339       n_y[i][j] = n_y[i][j] / norm - n[i] / norm * norm_y[j];
2340     }
2341   }
2342 
2343   nd = n[0] * d[0] + n[1] * d[1] + n[2] * d[2];
2344   for (PetscInt i=0; i<3; i++) nd_y[i] = n[i] + n_y[0][i] * d[0] + n_y[1][i] * d[1] + n_y[2][i] * d[2];
2345 
2346   res[0] = f;
2347   res[1] = d[1] - 2 * n[1] * nd;
2348   res[2] = d[2] - 2 * n[2] * nd;
2349   // J[j][i] is J_{ij} (column major)
2350   for (PetscInt j=0; j<3; j++) {
2351     J[0 + j*3] = grad[j];
2352     J[1 + j*3] = (j == 1)*1. - 2 * (n_y[1][j] * nd + n[1] * nd_y[j]);
2353     J[2 + j*3] = (j == 2)*1. - 2 * (n_y[2][j] * nd + n[2] * nd_y[j]);
2354   }
2355 }
2356 
2357 /*
2358    Project x to the nearest point on the implicit surface using Newton's method.
2359 */
2360 static PetscErrorCode TPSNearestPoint(TPSEvaluateFunc feval, PetscScalar x[])
2361 {
2362   PetscScalar y[3] = {x[0], x[1], x[2]}; // Initial guess
2363 
2364   PetscFunctionBegin;
2365   for (PetscInt iter=0; iter<10; iter++) {
2366     PetscScalar res[3], J[9];
2367     PetscReal resnorm;
2368     TPSNearestPointResJac(feval, x, y, res, J);
2369     resnorm = PetscSqrtReal(PetscSqr(PetscRealPart(res[0])) + PetscSqr(PetscRealPart(res[1])) + PetscSqr(PetscRealPart(res[2])));
2370     if (0) { // Turn on this monitor if you need to confirm quadratic convergence
2371       PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%" PetscInt_FMT "] res [%g %g %g]\n", iter, (double)PetscRealPart(res[0]), (double)PetscRealPart(res[1]), (double)PetscRealPart(res[2])));
2372     }
2373     if (resnorm < PETSC_SMALL) break;
2374 
2375     // Take the Newton step
2376     PetscCall(PetscKernel_A_gets_inverse_A_3(J, 0., PETSC_FALSE, NULL));
2377     PetscKernel_v_gets_v_minus_A_times_w_3(y, J, res);
2378   }
2379   for (PetscInt i=0; i<3; i++) x[i] = y[i];
2380   PetscFunctionReturn(0);
2381 }
2382 
2383 const char *const DMPlexTPSTypes[] = {"SCHWARZ_P", "GYROID", "DMPlexTPSType", "DMPLEX_TPS_", NULL};
2384 
2385 static PetscErrorCode DMPlexCreateTPSMesh_Internal(DM dm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness)
2386 {
2387   PetscMPIInt rank;
2388   PetscInt topoDim = 2, spaceDim = 3, numFaces = 0, numVertices = 0, numEdges = 0;
2389   PetscInt (*edges)[2] = NULL, *edgeSets = NULL;
2390   PetscInt *cells_flat = NULL;
2391   PetscReal *vtxCoords = NULL;
2392   TPSEvaluateFunc evalFunc = NULL;
2393   PetscSimplePointFunc normalFunc = NULL;
2394   DMLabel label;
2395 
2396   PetscFunctionBegin;
2397   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2398   PetscCheck((layers != 0) ^ (thickness == 0.), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Layers %" PetscInt_FMT " must be nonzero iff thickness %g is nonzero", layers, (double)thickness);
2399   switch (tpstype) {
2400   case DMPLEX_TPS_SCHWARZ_P:
2401     PetscCheck(!periodic || (periodic[0] == DM_BOUNDARY_NONE && periodic[1] == DM_BOUNDARY_NONE && periodic[2] == DM_BOUNDARY_NONE), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Schwarz P does not support periodic meshes");
2402     if (rank == 0) {
2403       PetscInt (*cells)[6][4][4] = NULL; // [junction, junction-face, cell, conn]
2404       PetscInt Njunctions = 0, Ncuts = 0, Npipes[3], vcount;
2405       PetscReal L = 1;
2406 
2407       Npipes[0] = (extent[0] + 1) * extent[1] * extent[2];
2408       Npipes[1] = extent[0] * (extent[1] + 1) * extent[2];
2409       Npipes[2] = extent[0] * extent[1] * (extent[2] + 1);
2410       Njunctions = extent[0] * extent[1] * extent[2];
2411       Ncuts = 2 * (extent[0] * extent[1] + extent[1] * extent[2] + extent[2] * extent[0]);
2412       numVertices = 4 * (Npipes[0] + Npipes[1] + Npipes[2]) + 8 * Njunctions;
2413       PetscCall(PetscMalloc1(3*numVertices, &vtxCoords));
2414       PetscCall(PetscMalloc1(Njunctions, &cells));
2415       PetscCall(PetscMalloc1(Ncuts*4, &edges));
2416       PetscCall(PetscMalloc1(Ncuts*4, &edgeSets));
2417       // x-normal pipes
2418       vcount = 0;
2419       for (PetscInt i=0; i<extent[0]+1; i++) {
2420         for (PetscInt j=0; j<extent[1]; j++) {
2421           for (PetscInt k=0; k<extent[2]; k++) {
2422             for (PetscInt l=0; l<4; l++) {
2423               vtxCoords[vcount++] = (2*i - 1) * L;
2424               vtxCoords[vcount++] = 2 * j * L + PetscCosReal((2*l + 1) * PETSC_PI / 4) * L / 2;
2425               vtxCoords[vcount++] = 2 * k * L + PetscSinReal((2*l + 1) * PETSC_PI / 4) * L / 2;
2426             }
2427           }
2428         }
2429       }
2430       // y-normal pipes
2431       for (PetscInt i=0; i<extent[0]; i++) {
2432         for (PetscInt j=0; j<extent[1]+1; j++) {
2433           for (PetscInt k=0; k<extent[2]; k++) {
2434             for (PetscInt l=0; l<4; l++) {
2435               vtxCoords[vcount++] = 2 * i * L + PetscSinReal((2*l + 1) * PETSC_PI / 4) * L / 2;
2436               vtxCoords[vcount++] = (2*j - 1) * L;
2437               vtxCoords[vcount++] = 2 * k * L + PetscCosReal((2*l + 1) * PETSC_PI / 4) * L / 2;
2438             }
2439           }
2440         }
2441       }
2442       // z-normal pipes
2443       for (PetscInt i=0; i<extent[0]; i++) {
2444         for (PetscInt j=0; j<extent[1]; j++) {
2445           for (PetscInt k=0; k<extent[2]+1; k++) {
2446             for (PetscInt l=0; l<4; l++) {
2447               vtxCoords[vcount++] = 2 * i * L + PetscCosReal((2*l + 1) * PETSC_PI / 4) * L / 2;
2448               vtxCoords[vcount++] = 2 * j * L + PetscSinReal((2*l + 1) * PETSC_PI / 4) * L / 2;
2449               vtxCoords[vcount++] = (2*k - 1) * L;
2450             }
2451           }
2452         }
2453       }
2454       // junctions
2455       for (PetscInt i=0; i<extent[0]; i++) {
2456         for (PetscInt j=0; j<extent[1]; j++) {
2457           for (PetscInt k=0; k<extent[2]; k++) {
2458             const PetscInt J = (i*extent[1] + j)*extent[2] + k, Jvoff = (Npipes[0] + Npipes[1] + Npipes[2])*4 + J*8;
2459             PetscCheck(vcount / 3 == Jvoff, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected vertex count");
2460             for (PetscInt ii=0; ii<2; ii++) {
2461               for (PetscInt jj=0; jj<2; jj++) {
2462                 for (PetscInt kk=0; kk<2; kk++) {
2463                   double Ls = (1 - sqrt(2) / 4) * L;
2464                   vtxCoords[vcount++] = 2*i*L + (2*ii-1) * Ls;
2465                   vtxCoords[vcount++] = 2*j*L + (2*jj-1) * Ls;
2466                   vtxCoords[vcount++] = 2*k*L + (2*kk-1) * Ls;
2467                 }
2468               }
2469             }
2470             const PetscInt jfaces[3][2][4] = {
2471               {{3,1,0,2}, {7,5,4,6}}, // x-aligned
2472               {{5,4,0,1}, {7,6,2,3}}, // y-aligned
2473               {{6,2,0,4}, {7,3,1,5}}  // z-aligned
2474             };
2475             const PetscInt pipe_lo[3] = { // vertex numbers of pipes
2476               ((i * extent[1] + j) * extent[2] + k)*4,
2477               ((i * (extent[1] + 1) + j) * extent[2] + k + Npipes[0])*4,
2478               ((i * extent[1] + j) * (extent[2]+1) + k + Npipes[0] + Npipes[1])*4
2479             };
2480             const PetscInt pipe_hi[3] = { // vertex numbers of pipes
2481               (((i + 1) * extent[1] + j) * extent[2] + k)*4,
2482               ((i * (extent[1] + 1) + j + 1) * extent[2] + k + Npipes[0])*4,
2483               ((i * extent[1] + j) * (extent[2]+1) + k + 1 + Npipes[0] + Npipes[1])*4
2484             };
2485             for (PetscInt dir=0; dir<3; dir++) { // x,y,z
2486               const PetscInt ijk[3] = {i, j, k};
2487               for (PetscInt l=0; l<4; l++) { // rotations
2488                 cells[J][dir*2+0][l][0] = pipe_lo[dir] + l;
2489                 cells[J][dir*2+0][l][1] = Jvoff + jfaces[dir][0][l];
2490                 cells[J][dir*2+0][l][2] = Jvoff + jfaces[dir][0][(l-1+4)%4];
2491                 cells[J][dir*2+0][l][3] = pipe_lo[dir] + (l-1+4)%4;
2492                 cells[J][dir*2+1][l][0] = Jvoff + jfaces[dir][1][l];
2493                 cells[J][dir*2+1][l][1] = pipe_hi[dir] + l;
2494                 cells[J][dir*2+1][l][2] = pipe_hi[dir] + (l-1+4)%4;
2495                 cells[J][dir*2+1][l][3] = Jvoff + jfaces[dir][1][(l-1+4)%4];
2496                 if (ijk[dir] == 0) {
2497                   edges[numEdges][0] = pipe_lo[dir] + l;
2498                   edges[numEdges][1] = pipe_lo[dir] + (l+1) % 4;
2499                   edgeSets[numEdges] = dir*2 + 1;
2500                   numEdges++;
2501                 }
2502                 if (ijk[dir] + 1 == extent[dir]) {
2503                   edges[numEdges][0] = pipe_hi[dir] + l;
2504                   edges[numEdges][1] = pipe_hi[dir] + (l+1) % 4;
2505                   edgeSets[numEdges] = dir*2 + 2;
2506                   numEdges++;
2507                 }
2508               }
2509             }
2510           }
2511         }
2512       }
2513       PetscCheck(numEdges == Ncuts * 4, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Edge count %" PetscInt_FMT " incompatible with number of cuts %" PetscInt_FMT, numEdges, Ncuts);
2514       numFaces = 24 * Njunctions;
2515       cells_flat = cells[0][0][0];
2516     }
2517     evalFunc = TPSEvaluate_SchwarzP;
2518     normalFunc = TPSExtrudeNormalFunc_SchwarzP;
2519     break;
2520   case DMPLEX_TPS_GYROID:
2521     if (rank == 0) {
2522       // This is a coarse mesh approximation of the gyroid shifted to being the zero of the level set
2523       //
2524       //     sin(pi*x)*cos(pi*(y+1/2)) + sin(pi*(y+1/2))*cos(pi*(z+1/4)) + sin(pi*(z+1/4))*cos(x)
2525       //
2526       // on the cell [0,2]^3.
2527       //
2528       // Think about dividing that cell into four columns, and focus on the column [0,1]x[0,1]x[0,2].
2529       // If you looked at the gyroid in that column at different slices of z you would see that it kind of spins
2530       // like a boomerang:
2531       //
2532       //     z = 0          z = 1/4        z = 1/2        z = 3/4     //
2533       //     -----          -------        -------        -------     //
2534       //                                                              //
2535       //     +       +      +       +      +       +      +   \   +   //
2536       //      \                                   /            \      //
2537       //       \            `-_   _-'            /              }     //
2538       //        *-_            `-'            _-'              /      //
2539       //     +     `-+      +       +      +-'     +      +   /   +   //
2540       //                                                              //
2541       //                                                              //
2542       //     z = 1          z = 5/4        z = 3/2        z = 7/4     //
2543       //     -----          -------        -------        -------     //
2544       //                                                              //
2545       //     +-_     +      +       +      +     _-+      +   /   +   //
2546       //        `-_            _-_            _-`            /        //
2547       //           \        _-'   `-_        /              {         //
2548       //            \                       /                \        //
2549       //     +       +      +       +      +       +      +   \   +   //
2550       //
2551       //
2552       // This course mesh approximates each of these slices by two line segments,
2553       // and then connects the segments in consecutive layers with quadrilateral faces.
2554       // All of the end points of the segments are multiples of 1/4 except for the
2555       // point * in the picture for z = 0 above and the similar points in other layers.
2556       // That point is at (gamma, gamma, 0), where gamma is calculated below.
2557       //
2558       // The column  [1,2]x[1,2]x[0,2] looks the same as this column;
2559       // The columns [1,2]x[0,1]x[0,2] and [0,1]x[1,2]x[0,2] are mirror images.
2560       //
2561       // As for how this method turned into the names given to the vertices:
2562       // that was not systematic, it was just the way it worked out in my handwritten notes.
2563 
2564       PetscInt facesPerBlock = 64;
2565       PetscInt vertsPerBlock = 56;
2566       PetscInt extentPlus[3];
2567       PetscInt numBlocks, numBlocksPlus;
2568       const PetscInt A =  0,   B =  1,   C =  2,   D =  3,   E =  4,   F =  5,   G =  6,   H =  7,
2569         II =  8,   J =  9,   K = 10,   L = 11,   M = 12,   N = 13,   O = 14,   P = 15,
2570         Q = 16,   R = 17,   S = 18,   T = 19,   U = 20,   V = 21,   W = 22,   X = 23,
2571         Y = 24,   Z = 25,  Ap = 26,  Bp = 27,  Cp = 28,  Dp = 29,  Ep = 30,  Fp = 31,
2572         Gp = 32,  Hp = 33,  Ip = 34,  Jp = 35,  Kp = 36,  Lp = 37,  Mp = 38,  Np = 39,
2573         Op = 40,  Pp = 41,  Qp = 42,  Rp = 43,  Sp = 44,  Tp = 45,  Up = 46,  Vp = 47,
2574         Wp = 48,  Xp = 49,  Yp = 50,  Zp = 51,  Aq = 52,  Bq = 53,  Cq = 54,  Dq = 55;
2575       const PetscInt pattern[64][4] =
2576         { /* face to vertex within the coarse discretization of a single gyroid block */
2577           /* layer 0 */
2578           {A,C,K,G},{C,B,II,K},{D,A,H,L},{B+56*1,D,L,J},{E,B+56*1,J,N},{A+56*2,E,N,H+56*2},{F,A+56*2,G+56*2,M},{B,F,M,II},
2579           /* layer 1 */
2580           {G,K,Q,O},{K,II,P,Q},{L,H,O+56*1,R},{J,L,R,P},{N,J,P,S},{H+56*2,N,S,O+56*3},{M,G+56*2,O+56*2,T},{II,M,T,P},
2581           /* layer 2 */
2582           {O,Q,Y,U},{Q,P,W,Y},{R,O+56*1,U+56*1,Ap},{P,R,Ap,W},{S,P,X,Bp},{O+56*3,S,Bp,V+56*1},{T,O+56*2,V,Z},{P,T,Z,X},
2583           /* layer 3 */
2584           {U,Y,Ep,Dp},{Y,W,Cp,Ep},{Ap,U+56*1,Dp+56*1,Gp},{W,Ap,Gp,Cp},{Bp,X,Cp+56*2,Fp},{V+56*1,Bp,Fp,Dp+56*1},{Z,V,Dp,Hp},{X,Z,Hp,Cp+56*2},
2585           /* layer 4 */
2586           {Dp,Ep,Mp,Kp},{Ep,Cp,Ip,Mp},{Gp,Dp+56*1,Lp,Np},{Cp,Gp,Np,Jp},{Fp,Cp+56*2,Jp+56*2,Pp},{Dp+56*1,Fp,Pp,Lp},{Hp,Dp,Kp,Op},{Cp+56*2,Hp,Op,Ip+56*2},
2587           /* layer 5 */
2588           {Kp,Mp,Sp,Rp},{Mp,Ip,Qp,Sp},{Np,Lp,Rp,Tp},{Jp,Np,Tp,Qp+56*1},{Pp,Jp+56*2,Qp+56*3,Up},{Lp,Pp,Up,Rp},{Op,Kp,Rp,Vp},{Ip+56*2,Op,Vp,Qp+56*2},
2589           /* layer 6 */
2590           {Rp,Sp,Aq,Yp},{Sp,Qp,Wp,Aq},{Tp,Rp,Yp,Cq},{Qp+56*1,Tp,Cq,Wp+56*1},{Up,Qp+56*3,Xp+56*1,Dq},{Rp,Up,Dq,Zp},{Vp,Rp,Zp,Bq},{Qp+56*2,Vp,Bq,Xp},
2591           /* layer 7 (the top is the periodic image of the bottom of layer 0) */
2592           {Yp,Aq,C+56*4,A+56*4},{Aq,Wp,B+56*4,C+56*4},{Cq,Yp,A+56*4,D+56*4},{Wp+56*1,Cq,D+56*4,B+56*5},{Dq,Xp+56*1,B+56*5,E+56*4},{Zp,Dq,E+56*4,A+56*6},{Bq,Zp,A+56*6,F+56*4},{Xp,Bq,F+56*4,B+56*4}
2593         };
2594       const PetscReal gamma = PetscAcosReal((PetscSqrtReal(3.)-1.) / PetscSqrtReal(2.)) / PETSC_PI;
2595       const PetscReal patternCoords[56][3] =
2596         {
2597           /* A  */ {1.,0.,0.},
2598           /* B  */ {0.,1.,0.},
2599           /* C  */ {gamma,gamma,0.},
2600           /* D  */ {1+gamma,1-gamma,0.},
2601           /* E  */ {2-gamma,2-gamma,0.},
2602           /* F  */ {1-gamma,1+gamma,0.},
2603 
2604           /* G  */ {.5,0,.25},
2605           /* H  */ {1.5,0.,.25},
2606           /* II */ {.5,1.,.25},
2607           /* J  */ {1.5,1.,.25},
2608           /* K  */ {.25,.5,.25},
2609           /* L  */ {1.25,.5,.25},
2610           /* M  */ {.75,1.5,.25},
2611           /* N  */ {1.75,1.5,.25},
2612 
2613           /* O  */ {0.,0.,.5},
2614           /* P  */ {1.,1.,.5},
2615           /* Q  */ {gamma,1-gamma,.5},
2616           /* R  */ {1+gamma,gamma,.5},
2617           /* S  */ {2-gamma,1+gamma,.5},
2618           /* T  */ {1-gamma,2-gamma,.5},
2619 
2620           /* U  */ {0.,.5,.75},
2621           /* V  */ {0.,1.5,.75},
2622           /* W  */ {1.,.5,.75},
2623           /* X  */ {1.,1.5,.75},
2624           /* Y  */ {.5,.75,.75},
2625           /* Z  */ {.5,1.75,.75},
2626           /* Ap */ {1.5,.25,.75},
2627           /* Bp */ {1.5,1.25,.75},
2628 
2629           /* Cp */ {1.,0.,1.},
2630           /* Dp */ {0.,1.,1.},
2631           /* Ep */ {1-gamma,1-gamma,1.},
2632           /* Fp */ {1+gamma,1+gamma,1.},
2633           /* Gp */ {2-gamma,gamma,1.},
2634           /* Hp */ {gamma,2-gamma,1.},
2635 
2636           /* Ip */ {.5,0.,1.25},
2637           /* Jp */ {1.5,0.,1.25},
2638           /* Kp */ {.5,1.,1.25},
2639           /* Lp */ {1.5,1.,1.25},
2640           /* Mp */ {.75,.5,1.25},
2641           /* Np */ {1.75,.5,1.25},
2642           /* Op */ {.25,1.5,1.25},
2643           /* Pp */ {1.25,1.5,1.25},
2644 
2645           /* Qp */ {0.,0.,1.5},
2646           /* Rp */ {1.,1.,1.5},
2647           /* Sp */ {1-gamma,gamma,1.5},
2648           /* Tp */ {2-gamma,1-gamma,1.5},
2649           /* Up */ {1+gamma,2-gamma,1.5},
2650           /* Vp */ {gamma,1+gamma,1.5},
2651 
2652           /* Wp */ {0.,.5,1.75},
2653           /* Xp */ {0.,1.5,1.75},
2654           /* Yp */ {1.,.5,1.75},
2655           /* Zp */ {1.,1.5,1.75},
2656           /* Aq */ {.5,.25,1.75},
2657           /* Bq */ {.5,1.25,1.75},
2658           /* Cq */ {1.5,.75,1.75},
2659           /* Dq */ {1.5,1.75,1.75},
2660         };
2661       PetscInt  (*cells)[64][4] = NULL;
2662       PetscBool *seen;
2663       PetscInt  *vertToTrueVert;
2664       PetscInt  count;
2665 
2666       for (PetscInt i = 0; i < 3; i++) extentPlus[i]  = extent[i] + 1;
2667       numBlocks = 1;
2668       for (PetscInt i = 0; i < 3; i++)     numBlocks *= extent[i];
2669       numBlocksPlus = 1;
2670       for (PetscInt i = 0; i < 3; i++) numBlocksPlus *= extentPlus[i];
2671       numFaces = numBlocks * facesPerBlock;
2672       PetscCall(PetscMalloc1(numBlocks, &cells));
2673       PetscCall(PetscCalloc1(numBlocksPlus * vertsPerBlock,&seen));
2674       for (PetscInt k = 0; k < extent[2]; k++) {
2675         for (PetscInt j = 0; j < extent[1]; j++) {
2676           for (PetscInt i = 0; i < extent[0]; i++) {
2677             for (PetscInt f = 0; f < facesPerBlock; f++) {
2678               for (PetscInt v = 0; v < 4; v++) {
2679                 PetscInt vertRaw = pattern[f][v];
2680                 PetscInt blockidx = vertRaw / 56;
2681                 PetscInt patternvert = vertRaw % 56;
2682                 PetscInt xplus = (blockidx & 1);
2683                 PetscInt yplus = (blockidx & 2) >> 1;
2684                 PetscInt zplus = (blockidx & 4) >> 2;
2685                 PetscInt zcoord = (periodic && periodic[2] == DM_BOUNDARY_PERIODIC) ? ((k + zplus) % extent[2]) : (k + zplus);
2686                 PetscInt ycoord = (periodic && periodic[1] == DM_BOUNDARY_PERIODIC) ? ((j + yplus) % extent[1]) : (j + yplus);
2687                 PetscInt xcoord = (periodic && periodic[0] == DM_BOUNDARY_PERIODIC) ? ((i + xplus) % extent[0]) : (i + xplus);
2688                 PetscInt vert = ((zcoord * extentPlus[1] + ycoord) * extentPlus[0] + xcoord) * 56 + patternvert;
2689 
2690                 cells[(k * extent[1] + j) * extent[0] + i][f][v] = vert;
2691                 seen[vert] = PETSC_TRUE;
2692               }
2693             }
2694           }
2695         }
2696       }
2697       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) if (seen[i]) numVertices++;
2698       count = 0;
2699       PetscCall(PetscMalloc1(numBlocksPlus * vertsPerBlock, &vertToTrueVert));
2700       PetscCall(PetscMalloc1(numVertices * 3, &vtxCoords));
2701       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) vertToTrueVert[i] = -1;
2702       for (PetscInt k = 0; k < extentPlus[2]; k++) {
2703         for (PetscInt j = 0; j < extentPlus[1]; j++) {
2704           for (PetscInt i = 0; i < extentPlus[0]; i++) {
2705             for (PetscInt v = 0; v < vertsPerBlock; v++) {
2706               PetscInt vIdx = ((k * extentPlus[1] + j) * extentPlus[0] + i) * vertsPerBlock + v;
2707 
2708               if (seen[vIdx]) {
2709                 PetscInt thisVert;
2710 
2711                 vertToTrueVert[vIdx] = thisVert = count++;
2712 
2713                 for (PetscInt d = 0; d < 3; d++) vtxCoords[3 * thisVert + d] = patternCoords[v][d];
2714                 vtxCoords[3 * thisVert + 0] += i * 2;
2715                 vtxCoords[3 * thisVert + 1] += j * 2;
2716                 vtxCoords[3 * thisVert + 2] += k * 2;
2717               }
2718             }
2719           }
2720         }
2721       }
2722       for (PetscInt i = 0; i < numBlocks; i++) {
2723         for (PetscInt f = 0; f < facesPerBlock; f++) {
2724           for (PetscInt v = 0; v < 4; v++) {
2725             cells[i][f][v] = vertToTrueVert[cells[i][f][v]];
2726           }
2727         }
2728       }
2729       PetscCall(PetscFree(vertToTrueVert));
2730       PetscCall(PetscFree(seen));
2731       cells_flat = cells[0][0];
2732       numEdges = 0;
2733       for (PetscInt i = 0; i < numFaces; i++) {
2734         for (PetscInt e = 0; e < 4; e++) {
2735           PetscInt ev[] = {cells_flat[i*4 + e], cells_flat[i*4 + ((e+1)%4)]};
2736           const PetscReal *evCoords[] = {&vtxCoords[3*ev[0]], &vtxCoords[3*ev[1]]};
2737 
2738           for (PetscInt d = 0; d < 3; d++) {
2739             if (!periodic || periodic[0] != DM_BOUNDARY_PERIODIC) {
2740               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) numEdges++;
2741               if (evCoords[0][d] == 2.*extent[d] && evCoords[1][d] == 2.*extent[d]) numEdges++;
2742             }
2743           }
2744         }
2745       }
2746       PetscCall(PetscMalloc1(numEdges, &edges));
2747       PetscCall(PetscMalloc1(numEdges, &edgeSets));
2748       for (PetscInt edge = 0, i = 0; i < numFaces; i++) {
2749         for (PetscInt e = 0; e < 4; e++) {
2750           PetscInt ev[] = {cells_flat[i*4 + e], cells_flat[i*4 + ((e+1)%4)]};
2751           const PetscReal *evCoords[] = {&vtxCoords[3*ev[0]], &vtxCoords[3*ev[1]]};
2752 
2753           for (PetscInt d = 0; d < 3; d++) {
2754             if (!periodic || periodic[d] != DM_BOUNDARY_PERIODIC) {
2755               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) {
2756                 edges[edge][0] = ev[0];
2757                 edges[edge][1] = ev[1];
2758                 edgeSets[edge++] = 2 * d;
2759               }
2760               if (evCoords[0][d] == 2.*extent[d] && evCoords[1][d] == 2.*extent[d]) {
2761                 edges[edge][0] = ev[0];
2762                 edges[edge][1] = ev[1];
2763                 edgeSets[edge++] = 2 * d + 1;
2764               }
2765             }
2766           }
2767         }
2768       }
2769     }
2770     evalFunc = TPSEvaluate_Gyroid;
2771     normalFunc = TPSExtrudeNormalFunc_Gyroid;
2772     break;
2773   }
2774 
2775   PetscCall(DMSetDimension(dm, topoDim));
2776   if (rank == 0) PetscCall(DMPlexBuildFromCellList(dm, numFaces, numVertices, 4, cells_flat));
2777   else           PetscCall(DMPlexBuildFromCellList(dm, 0, 0, 0, NULL));
2778   PetscCall(PetscFree(cells_flat));
2779   {
2780     DM idm;
2781     PetscCall(DMPlexInterpolate(dm, &idm));
2782     PetscCall(DMPlexReplace_Static(dm, &idm));
2783   }
2784   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, vtxCoords));
2785   else           PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, NULL));
2786   PetscCall(PetscFree(vtxCoords));
2787 
2788   PetscCall(DMCreateLabel(dm, "Face Sets"));
2789   PetscCall(DMGetLabel(dm, "Face Sets", &label));
2790   for (PetscInt e=0; e<numEdges; e++) {
2791     PetscInt njoin;
2792     const PetscInt *join, verts[] = {numFaces + edges[e][0], numFaces + edges[e][1]};
2793     PetscCall(DMPlexGetJoin(dm, 2, verts, &njoin, &join));
2794     PetscCheck(njoin == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected unique join of vertices %" PetscInt_FMT " and %" PetscInt_FMT, edges[e][0], edges[e][1]);
2795     PetscCall(DMLabelSetValue(label, join[0], edgeSets[e]));
2796     PetscCall(DMPlexRestoreJoin(dm, 2, verts, &njoin, &join));
2797   }
2798   PetscCall(PetscFree(edges));
2799   PetscCall(PetscFree(edgeSets));
2800   if (tps_distribute) {
2801     DM               pdm = NULL;
2802     PetscPartitioner part;
2803 
2804     PetscCall(DMPlexGetPartitioner(dm, &part));
2805     PetscCall(PetscPartitionerSetFromOptions(part));
2806     PetscCall(DMPlexDistribute(dm, 0, NULL, &pdm));
2807     if (pdm) {
2808       PetscCall(DMPlexReplace_Static(dm, &pdm));
2809     }
2810     // Do not auto-distribute again
2811     PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
2812   }
2813 
2814   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
2815   for (PetscInt refine=0; refine<refinements; refine++) {
2816     PetscInt m;
2817     DM dmf;
2818     Vec X;
2819     PetscScalar *x;
2820     PetscCall(DMRefine(dm, MPI_COMM_NULL, &dmf));
2821     PetscCall(DMPlexReplace_Static(dm, &dmf));
2822 
2823     PetscCall(DMGetCoordinatesLocal(dm, &X));
2824     PetscCall(VecGetLocalSize(X, &m));
2825     PetscCall(VecGetArray(X, &x));
2826     for (PetscInt i=0; i<m; i+=3) {
2827       PetscCall(TPSNearestPoint(evalFunc, &x[i]));
2828     }
2829     PetscCall(VecRestoreArray(X, &x));
2830   }
2831 
2832   // Face Sets has already been propagated to new vertices during refinement; this propagates to the initial vertices.
2833   PetscCall(DMGetLabel(dm, "Face Sets", &label));
2834   PetscCall(DMPlexLabelComplete(dm, label));
2835 
2836   if (thickness > 0) {
2837     DM edm,cdm,ecdm;
2838     DMPlexTransform tr;
2839     const char *prefix;
2840     PetscOptions options;
2841     // Code from DMPlexExtrude
2842     PetscCall(DMPlexTransformCreate(PetscObjectComm((PetscObject)dm), &tr));
2843     PetscCall(DMPlexTransformSetDM(tr, dm));
2844     PetscCall(DMPlexTransformSetType(tr, DMPLEXEXTRUDE));
2845     PetscCall(PetscObjectGetOptionsPrefix((PetscObject) dm, &prefix));
2846     PetscCall(PetscObjectSetOptionsPrefix((PetscObject) tr,  prefix));
2847     PetscCall(PetscObjectGetOptions((PetscObject) dm, &options));
2848     PetscCall(PetscObjectSetOptions((PetscObject) tr, options));
2849     PetscCall(DMPlexTransformExtrudeSetLayers(tr, layers));
2850     PetscCall(DMPlexTransformExtrudeSetThickness(tr, thickness));
2851     PetscCall(DMPlexTransformExtrudeSetTensor(tr, PETSC_FALSE));
2852     PetscCall(DMPlexTransformExtrudeSetSymmetric(tr, PETSC_TRUE));
2853     PetscCall(DMPlexTransformExtrudeSetNormalFunction(tr, normalFunc));
2854     PetscCall(DMPlexTransformSetFromOptions(tr));
2855     PetscCall(PetscObjectSetOptions((PetscObject) tr, NULL));
2856     PetscCall(DMPlexTransformSetUp(tr));
2857     PetscCall(PetscObjectViewFromOptions((PetscObject) tr, NULL, "-dm_plex_tps_transform_view"));
2858     PetscCall(DMPlexTransformApply(tr, dm, &edm));
2859     PetscCall(DMCopyDisc(dm, edm));
2860     PetscCall(DMGetCoordinateDM(dm, &cdm));
2861     PetscCall(DMGetCoordinateDM(edm, &ecdm));
2862     PetscCall(DMCopyDisc(cdm, ecdm));
2863     PetscCall(DMPlexTransformCreateDiscLabels(tr, edm));
2864     PetscCall(DMPlexTransformDestroy(&tr));
2865     if (edm) {
2866       ((DM_Plex *)edm->data)->printFEM = ((DM_Plex *)dm->data)->printFEM;
2867       ((DM_Plex *)edm->data)->printL2  = ((DM_Plex *)dm->data)->printL2;
2868     }
2869     PetscCall(DMPlexReplace_Static(dm, &edm));
2870   }
2871   PetscFunctionReturn(0);
2872 }
2873 
2874 /*@
2875   DMPlexCreateTPSMesh - Create a distributed, interpolated mesh of a triply-periodic surface
2876 
2877   Collective
2878 
2879   Input Parameters:
2880 + comm   - The communicator for the DM object
2881 . tpstype - Type of triply-periodic surface
2882 . extent - Array of length 3 containing number of periods in each direction
2883 . periodic - array of length 3 with periodicity, or NULL for non-periodic
2884 . tps_distribute - Distribute 2D manifold mesh prior to refinement and extrusion (more scalable)
2885 . refinements - Number of factor-of-2 refinements of 2D manifold mesh
2886 . layers - Number of cell layers extruded in normal direction
2887 - thickness - Thickness in normal direction
2888 
2889   Output Parameter:
2890 . dm  - The DM object
2891 
2892   Notes:
2893   This meshes the surface of the Schwarz P or Gyroid surfaces.  Schwarz P is is the simplest member of the triply-periodic minimal surfaces.
2894   https://en.wikipedia.org/wiki/Schwarz_minimal_surface#Schwarz_P_(%22Primitive%22) and can be cut with "clean" boundaries.
2895   The Gyroid (https://en.wikipedia.org/wiki/Gyroid) is another triply-periodic minimal surface with applications in additive manufacturing; it is much more difficult to "cut" since there are no planes of symmetry.
2896   Our implementation creates a very coarse mesh of the surface and refines (by 4-way splitting) as many times as requested.
2897   On each refinement, all vertices are projected to their nearest point on the surface.
2898   This projection could readily be extended to related surfaces.
2899 
2900   The face (edge) sets for the Schwarz P surface are numbered 1(-x), 2(+x), 3(-y), 4(+y), 5(-z), 6(+z).
2901   When the mesh is refined, "Face Sets" contain the new vertices (created during refinement).  Use DMPlexLabelComplete() to propagate to coarse-level vertices.
2902 
2903   References:
2904 . * - Maskery et al, Insights into the mechanical properties of several triply periodic minimal surface lattice structures made by polymer additive manufacturing, 2017. https://doi.org/10.1016/j.polymer.2017.11.049
2905 
2906   Developer Notes:
2907   The Gyroid mesh does not currently mark boundary sets.
2908 
2909   Level: beginner
2910 
2911 .seealso: `DMPlexCreateSphereMesh()`, `DMSetType()`, `DMCreate()`
2912 @*/
2913 PetscErrorCode DMPlexCreateTPSMesh(MPI_Comm comm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness, DM *dm)
2914 {
2915   PetscFunctionBegin;
2916   PetscCall(DMCreate(comm, dm));
2917   PetscCall(DMSetType(*dm, DMPLEX));
2918   PetscCall(DMPlexCreateTPSMesh_Internal(*dm, tpstype, extent, periodic, tps_distribute, refinements, layers, thickness));
2919   PetscFunctionReturn(0);
2920 }
2921 
2922 /*@
2923   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
2924 
2925   Collective
2926 
2927   Input Parameters:
2928 + comm    - The communicator for the DM object
2929 . dim     - The dimension
2930 . simplex - Use simplices, or tensor product cells
2931 - R       - The radius
2932 
2933   Output Parameter:
2934 . dm  - The DM object
2935 
2936   Level: beginner
2937 
2938 .seealso: `DMPlexCreateBallMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2939 @*/
2940 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscReal R, DM *dm)
2941 {
2942   PetscFunctionBegin;
2943   PetscValidPointer(dm, 5);
2944   PetscCall(DMCreate(comm, dm));
2945   PetscCall(DMSetType(*dm, DMPLEX));
2946   PetscCall(DMPlexCreateSphereMesh_Internal(*dm, dim, simplex, R));
2947   PetscFunctionReturn(0);
2948 }
2949 
2950 static PetscErrorCode DMPlexCreateBallMesh_Internal(DM dm, PetscInt dim, PetscReal R)
2951 {
2952   DM             sdm, vol;
2953   DMLabel        bdlabel;
2954 
2955   PetscFunctionBegin;
2956   PetscCall(DMCreate(PetscObjectComm((PetscObject) dm), &sdm));
2957   PetscCall(DMSetType(sdm, DMPLEX));
2958   PetscCall(PetscObjectSetOptionsPrefix((PetscObject) sdm, "bd_"));
2959   PetscCall(DMPlexCreateSphereMesh_Internal(sdm, dim-1, PETSC_TRUE, R));
2960   PetscCall(DMSetFromOptions(sdm));
2961   PetscCall(DMViewFromOptions(sdm, NULL, "-dm_view"));
2962   PetscCall(DMPlexGenerate(sdm, NULL, PETSC_TRUE, &vol));
2963   PetscCall(DMDestroy(&sdm));
2964   PetscCall(DMPlexReplace_Static(dm, &vol));
2965   PetscCall(DMCreateLabel(dm, "marker"));
2966   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
2967   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
2968   PetscCall(DMPlexLabelComplete(dm, bdlabel));
2969   PetscFunctionReturn(0);
2970 }
2971 
2972 /*@
2973   DMPlexCreateBallMesh - Creates a simplex mesh on the d-dimensional ball, B^d.
2974 
2975   Collective
2976 
2977   Input Parameters:
2978 + comm  - The communicator for the DM object
2979 . dim   - The dimension
2980 - R     - The radius
2981 
2982   Output Parameter:
2983 . dm  - The DM object
2984 
2985   Options Database Keys:
2986 - bd_dm_refine - This will refine the surface mesh preserving the sphere geometry
2987 
2988   Level: beginner
2989 
2990 .seealso: `DMPlexCreateSphereMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2991 @*/
2992 PetscErrorCode DMPlexCreateBallMesh(MPI_Comm comm, PetscInt dim, PetscReal R, DM *dm)
2993 {
2994   PetscFunctionBegin;
2995   PetscCall(DMCreate(comm, dm));
2996   PetscCall(DMSetType(*dm, DMPLEX));
2997   PetscCall(DMPlexCreateBallMesh_Internal(*dm, dim, R));
2998   PetscFunctionReturn(0);
2999 }
3000 
3001 static PetscErrorCode DMPlexCreateReferenceCell_Internal(DM rdm, DMPolytopeType ct)
3002 {
3003   PetscFunctionBegin;
3004   switch (ct) {
3005     case DM_POLYTOPE_POINT:
3006     {
3007       PetscInt    numPoints[1]        = {1};
3008       PetscInt    coneSize[1]         = {0};
3009       PetscInt    cones[1]            = {0};
3010       PetscInt    coneOrientations[1] = {0};
3011       PetscScalar vertexCoords[1]     = {0.0};
3012 
3013       PetscCall(DMSetDimension(rdm, 0));
3014       PetscCall(DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3015     }
3016     break;
3017     case DM_POLYTOPE_SEGMENT:
3018     {
3019       PetscInt    numPoints[2]        = {2, 1};
3020       PetscInt    coneSize[3]         = {2, 0, 0};
3021       PetscInt    cones[2]            = {1, 2};
3022       PetscInt    coneOrientations[2] = {0, 0};
3023       PetscScalar vertexCoords[2]     = {-1.0,  1.0};
3024 
3025       PetscCall(DMSetDimension(rdm, 1));
3026       PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3027     }
3028     break;
3029     case DM_POLYTOPE_POINT_PRISM_TENSOR:
3030     {
3031       PetscInt    numPoints[2]        = {2, 1};
3032       PetscInt    coneSize[3]         = {2, 0, 0};
3033       PetscInt    cones[2]            = {1, 2};
3034       PetscInt    coneOrientations[2] = {0, 0};
3035       PetscScalar vertexCoords[2]     = {-1.0,  1.0};
3036 
3037       PetscCall(DMSetDimension(rdm, 1));
3038       PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3039     }
3040     break;
3041     case DM_POLYTOPE_TRIANGLE:
3042     {
3043       PetscInt    numPoints[2]        = {3, 1};
3044       PetscInt    coneSize[4]         = {3, 0, 0, 0};
3045       PetscInt    cones[3]            = {1, 2, 3};
3046       PetscInt    coneOrientations[3] = {0, 0, 0};
3047       PetscScalar vertexCoords[6]     = {-1.0, -1.0,  1.0, -1.0,  -1.0, 1.0};
3048 
3049       PetscCall(DMSetDimension(rdm, 2));
3050       PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3051     }
3052     break;
3053     case DM_POLYTOPE_QUADRILATERAL:
3054     {
3055       PetscInt    numPoints[2]        = {4, 1};
3056       PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3057       PetscInt    cones[4]            = {1, 2, 3, 4};
3058       PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3059       PetscScalar vertexCoords[8]     = {-1.0, -1.0,  1.0, -1.0,  1.0, 1.0,  -1.0, 1.0};
3060 
3061       PetscCall(DMSetDimension(rdm, 2));
3062       PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3063     }
3064     break;
3065     case DM_POLYTOPE_SEG_PRISM_TENSOR:
3066     {
3067       PetscInt    numPoints[2]        = {4, 1};
3068       PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3069       PetscInt    cones[4]            = {1, 2, 3, 4};
3070       PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3071       PetscScalar vertexCoords[8]     = {-1.0, -1.0,  1.0, -1.0,  -1.0, 1.0,  1.0, 1.0};
3072 
3073       PetscCall(DMSetDimension(rdm, 2));
3074       PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3075     }
3076     break;
3077     case DM_POLYTOPE_TETRAHEDRON:
3078     {
3079       PetscInt    numPoints[2]        = {4, 1};
3080       PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3081       PetscInt    cones[4]            = {1, 2, 3, 4};
3082       PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3083       PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0,  -1.0, 1.0, -1.0,  1.0, -1.0, -1.0,  -1.0, -1.0, 1.0};
3084 
3085       PetscCall(DMSetDimension(rdm, 3));
3086       PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3087     }
3088     break;
3089     case DM_POLYTOPE_HEXAHEDRON:
3090     {
3091       PetscInt    numPoints[2]        = {8, 1};
3092       PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3093       PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3094       PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3095       PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0,  -1.0,  1.0, -1.0,  1.0, 1.0, -1.0,   1.0, -1.0, -1.0,
3096                                          -1.0, -1.0,  1.0,   1.0, -1.0,  1.0,  1.0, 1.0,  1.0,  -1.0,  1.0,  1.0};
3097 
3098       PetscCall(DMSetDimension(rdm, 3));
3099       PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3100     }
3101     break;
3102     case DM_POLYTOPE_TRI_PRISM:
3103     {
3104       PetscInt    numPoints[2]        = {6, 1};
3105       PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3106       PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3107       PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3108       PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, -1.0,  1.0, -1.0,   1.0, -1.0, -1.0,
3109                                          -1.0, -1.0,  1.0,  1.0, -1.0,  1.0,  -1.0,  1.0,  1.0};
3110 
3111       PetscCall(DMSetDimension(rdm, 3));
3112       PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3113     }
3114     break;
3115     case DM_POLYTOPE_TRI_PRISM_TENSOR:
3116     {
3117       PetscInt    numPoints[2]        = {6, 1};
3118       PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3119       PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3120       PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3121       PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0,  1.0, -1.0, -1.0,  -1.0, 1.0, -1.0,
3122                                          -1.0, -1.0,  1.0,  1.0, -1.0,  1.0,  -1.0, 1.0,  1.0};
3123 
3124       PetscCall(DMSetDimension(rdm, 3));
3125       PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3126     }
3127     break;
3128     case DM_POLYTOPE_QUAD_PRISM_TENSOR:
3129     {
3130       PetscInt    numPoints[2]        = {8, 1};
3131       PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3132       PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3133       PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3134       PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0,  1.0, -1.0, -1.0,  1.0, 1.0, -1.0,  -1.0, 1.0, -1.0,
3135                                          -1.0, -1.0,  1.0,  1.0, -1.0,  1.0,  1.0, 1.0,  1.0,  -1.0, 1.0,  1.0};
3136 
3137       PetscCall(DMSetDimension(rdm, 3));
3138       PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3139     }
3140     break;
3141     case DM_POLYTOPE_PYRAMID:
3142     {
3143       PetscInt    numPoints[2]        = {5, 1};
3144       PetscInt    coneSize[6]         = {5, 0, 0, 0, 0, 0};
3145       PetscInt    cones[5]            = {1, 2, 3, 4, 5};
3146       PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3147       PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0,  -1.0, 1.0, -1.0,  1.0, 1.0, -1.0,  1.0, -1.0, -1.0,
3148                                           0.0,  0.0,  1.0};
3149 
3150       PetscCall(DMSetDimension(rdm, 3));
3151       PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3152     }
3153     break;
3154     default: SETERRQ(PetscObjectComm((PetscObject) rdm), PETSC_ERR_ARG_WRONG, "Cannot create reference cell for cell type %s", DMPolytopeTypes[ct]);
3155   }
3156   {
3157     PetscInt Nv, v;
3158 
3159     /* Must create the celltype label here so that we do not automatically try to compute the types */
3160     PetscCall(DMCreateLabel(rdm, "celltype"));
3161     PetscCall(DMPlexSetCellType(rdm, 0, ct));
3162     PetscCall(DMPlexGetChart(rdm, NULL, &Nv));
3163     for (v = 1; v < Nv; ++v) PetscCall(DMPlexSetCellType(rdm, v, DM_POLYTOPE_POINT));
3164   }
3165   PetscCall(DMPlexInterpolateInPlace_Internal(rdm));
3166   PetscCall(PetscObjectSetName((PetscObject) rdm, DMPolytopeTypes[ct]));
3167   PetscFunctionReturn(0);
3168 }
3169 
3170 /*@
3171   DMPlexCreateReferenceCell - Create a DMPLEX with the appropriate FEM reference cell
3172 
3173   Collective
3174 
3175   Input Parameters:
3176 + comm - The communicator
3177 - ct   - The cell type of the reference cell
3178 
3179   Output Parameter:
3180 . refdm - The reference cell
3181 
3182   Level: intermediate
3183 
3184 .seealso: `DMPlexCreateReferenceCell()`, `DMPlexCreateBoxMesh()`
3185 @*/
3186 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, DMPolytopeType ct, DM *refdm)
3187 {
3188   PetscFunctionBegin;
3189   PetscCall(DMCreate(comm, refdm));
3190   PetscCall(DMSetType(*refdm, DMPLEX));
3191   PetscCall(DMPlexCreateReferenceCell_Internal(*refdm, ct));
3192   PetscFunctionReturn(0);
3193 }
3194 
3195 static PetscErrorCode DMPlexCreateBoundaryLabel_Private(DM dm, const char name[])
3196 {
3197   DM             plex;
3198   DMLabel        label;
3199   PetscBool      hasLabel;
3200 
3201   PetscFunctionBegin;
3202   PetscCall(DMHasLabel(dm, name, &hasLabel));
3203   if (hasLabel) PetscFunctionReturn(0);
3204   PetscCall(DMCreateLabel(dm, name));
3205   PetscCall(DMGetLabel(dm, name, &label));
3206   PetscCall(DMConvert(dm, DMPLEX, &plex));
3207   PetscCall(DMPlexMarkBoundaryFaces(plex, 1, label));
3208   PetscCall(DMPlexLabelComplete(plex, label));
3209   PetscCall(DMDestroy(&plex));
3210   PetscFunctionReturn(0);
3211 }
3212 
3213 const char * const DMPlexShapes[] = {"box", "box_surface", "ball", "sphere", "cylinder", "schwarz_p", "gyroid", "doublet", "unknown", "DMPlexShape", "DM_SHAPE_", NULL};
3214 
3215 static PetscErrorCode DMPlexCreateFromOptions_Internal(PetscOptionItems *PetscOptionsObject, PetscBool *useCoordSpace, DM dm)
3216 {
3217   DMPlexShape    shape = DM_SHAPE_BOX;
3218   DMPolytopeType cell  = DM_POLYTOPE_TRIANGLE;
3219   PetscInt       dim   = 2;
3220   PetscBool      simplex = PETSC_TRUE, interpolate = PETSC_TRUE, adjCone = PETSC_FALSE, adjClosure = PETSC_TRUE, refDomain = PETSC_FALSE;
3221   PetscBool      flg, flg2, fflg, bdfflg, nameflg;
3222   MPI_Comm       comm;
3223   char           filename[PETSC_MAX_PATH_LEN]   = "<unspecified>";
3224   char           bdFilename[PETSC_MAX_PATH_LEN] = "<unspecified>";
3225   char           plexname[PETSC_MAX_PATH_LEN]   = "";
3226 
3227   PetscFunctionBegin;
3228   PetscCall(PetscObjectGetComm((PetscObject) dm, &comm));
3229   /* TODO Turn this into a registration interface */
3230   PetscCall(PetscOptionsString("-dm_plex_filename", "File containing a mesh", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &fflg));
3231   PetscCall(PetscOptionsString("-dm_plex_boundary_filename", "File containing a mesh boundary", "DMPlexCreateFromFile", bdFilename, bdFilename, sizeof(bdFilename), &bdfflg));
3232   PetscCall(PetscOptionsString("-dm_plex_name", "Name of the mesh in the file", "DMPlexCreateFromFile", plexname, plexname, sizeof(plexname), &nameflg));
3233   PetscCall(PetscOptionsEnum("-dm_plex_cell", "Cell shape", "", DMPolytopeTypes, (PetscEnum) cell, (PetscEnum *) &cell, NULL));
3234   PetscCall(PetscOptionsBool("-dm_plex_reference_cell_domain", "Use a reference cell domain", "", refDomain, &refDomain, NULL));
3235   PetscCall(PetscOptionsEnum("-dm_plex_shape", "Shape for built-in mesh", "", DMPlexShapes, (PetscEnum) shape, (PetscEnum *) &shape, &flg));
3236   PetscCall(PetscOptionsBoundedInt("-dm_plex_dim", "Topological dimension of the mesh", "DMGetDimension", dim, &dim, &flg, 0));
3237   PetscCheck(!(dim < 0) && !(dim > 3),comm, PETSC_ERR_ARG_OUTOFRANGE, "Dimension %" PetscInt_FMT " should be in [1, 3]", dim);
3238   PetscCall(PetscOptionsBool("-dm_plex_simplex", "Mesh cell shape", "", simplex,  &simplex, &flg));
3239   PetscCall(PetscOptionsBool("-dm_plex_interpolate", "Flag to create edges and faces automatically", "", interpolate, &interpolate, &flg));
3240   PetscCall(PetscOptionsBool("-dm_plex_adj_cone", "Set adjacency direction", "DMSetBasicAdjacency", adjCone,  &adjCone, &flg));
3241   PetscCall(PetscOptionsBool("-dm_plex_adj_closure", "Set adjacency size", "DMSetBasicAdjacency", adjClosure,  &adjClosure, &flg2));
3242   if (flg || flg2) PetscCall(DMSetBasicAdjacency(dm, adjCone, adjClosure));
3243 
3244   switch (cell) {
3245     case DM_POLYTOPE_POINT:
3246     case DM_POLYTOPE_SEGMENT:
3247     case DM_POLYTOPE_POINT_PRISM_TENSOR:
3248     case DM_POLYTOPE_TRIANGLE:
3249     case DM_POLYTOPE_QUADRILATERAL:
3250     case DM_POLYTOPE_TETRAHEDRON:
3251     case DM_POLYTOPE_HEXAHEDRON:
3252       *useCoordSpace = PETSC_TRUE;break;
3253     default: *useCoordSpace = PETSC_FALSE;break;
3254   }
3255 
3256   if (fflg) {
3257     DM dmnew;
3258 
3259     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject) dm), filename, plexname, interpolate, &dmnew));
3260     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3261     PetscCall(DMPlexReplace_Static(dm, &dmnew));
3262   } else if (refDomain) {
3263     PetscCall(DMPlexCreateReferenceCell_Internal(dm, cell));
3264   } else if (bdfflg) {
3265     DM bdm, dmnew;
3266 
3267     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject) dm), bdFilename, plexname, interpolate, &bdm));
3268     PetscCall(PetscObjectSetOptionsPrefix((PetscObject) bdm, "bd_"));
3269     PetscCall(DMSetFromOptions(bdm));
3270     PetscCall(DMPlexGenerate(bdm, NULL, interpolate, &dmnew));
3271     PetscCall(DMDestroy(&bdm));
3272     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3273     PetscCall(DMPlexReplace_Static(dm, &dmnew));
3274   } else {
3275     PetscCall(PetscObjectSetName((PetscObject) dm, DMPlexShapes[shape]));
3276     switch (shape) {
3277       case DM_SHAPE_BOX:
3278       {
3279         PetscInt       faces[3] = {0, 0, 0};
3280         PetscReal      lower[3] = {0, 0, 0};
3281         PetscReal      upper[3] = {1, 1, 1};
3282         DMBoundaryType bdt[3]   = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3283         PetscInt       i, n;
3284 
3285         n    = dim;
3286         for (i = 0; i < dim; ++i) faces[i] = (dim == 1 ? 1 : 4-dim);
3287         PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3288         n    = 3;
3289         PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3290         PetscCheck(!flg || !(n != dim),comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3291         n    = 3;
3292         PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3293         PetscCheck(!flg || !(n != dim),comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3294         n    = 3;
3295         PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *) bdt, &n, &flg));
3296         PetscCheck(!flg || !(n != dim),comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3297         switch (cell) {
3298           case DM_POLYTOPE_TRI_PRISM_TENSOR:
3299             PetscCall(DMPlexCreateWedgeBoxMesh_Internal(dm, faces, lower, upper, bdt));
3300             if (!interpolate) {
3301               DM udm;
3302 
3303               PetscCall(DMPlexUninterpolate(dm, &udm));
3304               PetscCall(DMPlexReplace_Static(dm, &udm));
3305             }
3306             break;
3307           default:
3308             PetscCall(DMPlexCreateBoxMesh_Internal(dm, dim, simplex, faces, lower, upper, bdt, interpolate));
3309             break;
3310         }
3311       }
3312       break;
3313       case DM_SHAPE_BOX_SURFACE:
3314       {
3315         PetscInt  faces[3] = {0, 0, 0};
3316         PetscReal lower[3] = {0, 0, 0};
3317         PetscReal upper[3] = {1, 1, 1};
3318         PetscInt  i, n;
3319 
3320         n    = dim+1;
3321         for (i = 0; i < dim+1; ++i) faces[i] = (dim+1 == 1 ? 1 : 4-(dim+1));
3322         PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3323         n    = 3;
3324         PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3325         PetscCheck(!flg || !(n != dim+1),comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim+1);
3326         n    = 3;
3327         PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3328         PetscCheck(!flg || !(n != dim+1),comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim+1);
3329         PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(dm, dim+1, faces, lower, upper, interpolate));
3330       }
3331       break;
3332       case DM_SHAPE_SPHERE:
3333       {
3334         PetscReal R = 1.0;
3335 
3336         PetscCall(PetscOptionsReal("-dm_plex_sphere_radius", "Radius of the sphere", "", R,  &R, &flg));
3337         PetscCall(DMPlexCreateSphereMesh_Internal(dm, dim, simplex, R));
3338       }
3339       break;
3340       case DM_SHAPE_BALL:
3341       {
3342         PetscReal R = 1.0;
3343 
3344         PetscCall(PetscOptionsReal("-dm_plex_ball_radius", "Radius of the ball", "", R,  &R, &flg));
3345         PetscCall(DMPlexCreateBallMesh_Internal(dm, dim, R));
3346       }
3347       break;
3348       case DM_SHAPE_CYLINDER:
3349       {
3350         DMBoundaryType bdt = DM_BOUNDARY_NONE;
3351         PetscInt       Nw  = 6;
3352 
3353         PetscCall(PetscOptionsEnum("-dm_plex_cylinder_bd", "Boundary type in the z direction", "", DMBoundaryTypes, (PetscEnum) bdt, (PetscEnum *) &bdt, NULL));
3354         PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_wedges", "Number of wedges around the cylinder", "", Nw, &Nw, NULL));
3355         switch (cell) {
3356           case DM_POLYTOPE_TRI_PRISM_TENSOR:
3357             PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(dm, Nw, interpolate));
3358             break;
3359           default:
3360             PetscCall(DMPlexCreateHexCylinderMesh_Internal(dm, bdt));
3361             break;
3362         }
3363       }
3364       break;
3365       case DM_SHAPE_SCHWARZ_P: // fallthrough
3366       case DM_SHAPE_GYROID:
3367       {
3368         PetscInt       extent[3] = {1,1,1}, refine = 0, layers = 0, three;
3369         PetscReal      thickness = 0.;
3370         DMBoundaryType periodic[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3371         DMPlexTPSType  tps_type = shape == DM_SHAPE_SCHWARZ_P ? DMPLEX_TPS_SCHWARZ_P : DMPLEX_TPS_GYROID;
3372         PetscBool      tps_distribute;
3373         PetscCall(PetscOptionsIntArray("-dm_plex_tps_extent", "Number of replicas for each of three dimensions", NULL, extent, (three=3, &three), NULL));
3374         PetscCall(PetscOptionsInt("-dm_plex_tps_refine", "Number of refinements", NULL, refine, &refine, NULL));
3375         PetscCall(PetscOptionsEnumArray("-dm_plex_tps_periodic", "Periodicity in each of three dimensions", NULL, DMBoundaryTypes, (PetscEnum*)periodic, (three=3, &three), NULL));
3376         PetscCall(PetscOptionsInt("-dm_plex_tps_layers", "Number of layers in volumetric extrusion (or zero to not extrude)", NULL, layers, &layers, NULL));
3377         PetscCall(PetscOptionsReal("-dm_plex_tps_thickness", "Thickness of volumetric extrusion", NULL, thickness, &thickness, NULL));
3378         PetscCall(DMPlexDistributeGetDefault(dm, &tps_distribute));
3379         PetscCall(PetscOptionsBool("-dm_plex_tps_distribute", "Distribute the 2D mesh prior to refinement and extrusion", NULL, tps_distribute, &tps_distribute, NULL));
3380         PetscCall(DMPlexCreateTPSMesh_Internal(dm, tps_type, extent, periodic, tps_distribute, refine, layers, thickness));
3381       }
3382       break;
3383       case DM_SHAPE_DOUBLET:
3384       {
3385         DM        dmnew;
3386         PetscReal rl = 0.0;
3387 
3388         PetscCall(PetscOptionsReal("-dm_plex_doublet_refinementlimit", "Refinement limit", NULL, rl, &rl, NULL));
3389         PetscCall(DMPlexCreateDoublet(PetscObjectComm((PetscObject)dm), dim, simplex, interpolate, rl, &dmnew));
3390         PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3391         PetscCall(DMPlexReplace_Static(dm, &dmnew));
3392       }
3393       break;
3394       default: SETERRQ(comm, PETSC_ERR_SUP, "Domain shape %s is unsupported", DMPlexShapes[shape]);
3395     }
3396   }
3397   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3398   if (!((PetscObject)dm)->name && nameflg) {
3399     PetscCall(PetscObjectSetName((PetscObject)dm, plexname));
3400   }
3401   PetscFunctionReturn(0);
3402 }
3403 
3404 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(PetscOptionItems *PetscOptionsObject, DM dm)
3405 {
3406   DM_Plex       *mesh = (DM_Plex*) dm->data;
3407   PetscBool      flg, flg2;
3408   char           bdLabel[PETSC_MAX_PATH_LEN];
3409 
3410   PetscFunctionBegin;
3411   /* Handle viewing */
3412   PetscCall(PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL));
3413   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fem", "Debug output level all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL,0));
3414   PetscCall(PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL));
3415   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL,0));
3416   PetscCall(DMMonitorSetFromOptions(dm, "-dm_plex_monitor_throughput", "Monitor the simulation throughput", "DMPlexMonitorThroughput", DMPlexMonitorThroughput, NULL, &flg));
3417   if (flg) PetscCall(PetscLogDefaultBegin());
3418   /* Labeling */
3419   PetscCall(PetscOptionsString("-dm_plex_boundary_label", "Label to mark the mesh boundary", "", bdLabel, bdLabel, sizeof(bdLabel), &flg));
3420   if (flg) PetscCall(DMPlexCreateBoundaryLabel_Private(dm, bdLabel));
3421   /* Point Location */
3422   PetscCall(PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL));
3423   /* Partitioning and distribution */
3424   PetscCall(PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL));
3425   /* Generation and remeshing */
3426   PetscCall(PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL));
3427   /* Projection behavior */
3428   PetscCall(PetscOptionsBoundedInt("-dm_plex_max_projection_height", "Maxmimum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL,0));
3429   PetscCall(PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL));
3430   /* Checking structure */
3431   {
3432     PetscBool   all = PETSC_FALSE;
3433 
3434     PetscCall(PetscOptionsBool("-dm_plex_check_all", "Perform all basic checks", "DMPlexCheck", PETSC_FALSE, &all, NULL));
3435     if (all) {
3436       PetscCall(DMPlexCheck(dm));
3437     } else {
3438       PetscCall(PetscOptionsBool("-dm_plex_check_symmetry", "Check that the adjacency information in the mesh is symmetric", "DMPlexCheckSymmetry", PETSC_FALSE, &flg, &flg2));
3439       if (flg && flg2) PetscCall(DMPlexCheckSymmetry(dm));
3440       PetscCall(PetscOptionsBool("-dm_plex_check_skeleton", "Check that each cell has the correct number of vertices (only for homogeneous simplex or tensor meshes)", "DMPlexCheckSkeleton", PETSC_FALSE, &flg, &flg2));
3441       if (flg && flg2) PetscCall(DMPlexCheckSkeleton(dm, 0));
3442       PetscCall(PetscOptionsBool("-dm_plex_check_faces", "Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type", "DMPlexCheckFaces", PETSC_FALSE, &flg, &flg2));
3443       if (flg && flg2) PetscCall(DMPlexCheckFaces(dm, 0));
3444       PetscCall(PetscOptionsBool("-dm_plex_check_geometry", "Check that cells have positive volume", "DMPlexCheckGeometry", PETSC_FALSE, &flg, &flg2));
3445       if (flg && flg2) PetscCall(DMPlexCheckGeometry(dm));
3446       PetscCall(PetscOptionsBool("-dm_plex_check_pointsf", "Check some necessary conditions for PointSF", "DMPlexCheckPointSF", PETSC_FALSE, &flg, &flg2));
3447       if (flg && flg2) PetscCall(DMPlexCheckPointSF(dm, NULL));
3448       PetscCall(PetscOptionsBool("-dm_plex_check_interface_cones", "Check points on inter-partition interfaces have conforming order of cone points", "DMPlexCheckInterfaceCones", PETSC_FALSE, &flg, &flg2));
3449       if (flg && flg2) PetscCall(DMPlexCheckInterfaceCones(dm));
3450     }
3451     PetscCall(PetscOptionsBool("-dm_plex_check_cell_shape", "Check cell shape", "DMPlexCheckCellShape", PETSC_FALSE, &flg, &flg2));
3452     if (flg && flg2) PetscCall(DMPlexCheckCellShape(dm, PETSC_TRUE, PETSC_DETERMINE));
3453   }
3454   {
3455     PetscReal scale = 1.0;
3456 
3457     PetscCall(PetscOptionsReal("-dm_plex_scale", "Scale factor for mesh coordinates", "DMPlexScale", scale, &scale, &flg));
3458     if (flg) {
3459       Vec coordinates, coordinatesLocal;
3460 
3461       PetscCall(DMGetCoordinates(dm, &coordinates));
3462       PetscCall(DMGetCoordinatesLocal(dm, &coordinatesLocal));
3463       PetscCall(VecScale(coordinates, scale));
3464       PetscCall(VecScale(coordinatesLocal, scale));
3465     }
3466   }
3467   PetscCall(PetscPartitionerSetFromOptions(mesh->partitioner));
3468   PetscFunctionReturn(0);
3469 }
3470 
3471 PetscErrorCode DMSetFromOptions_Overlap_Plex(PetscOptionItems *PetscOptionsObject, DM dm, PetscInt *overlap)
3472 {
3473   PetscInt  numOvLabels = 16, numOvExLabels = 16;
3474   char     *ovLabelNames[16], *ovExLabelNames[16];
3475   PetscInt  numOvValues = 16, numOvExValues = 16, l;
3476   PetscBool flg;
3477 
3478   PetscFunctionBegin;
3479   PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", *overlap, overlap, NULL, 0));
3480   PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_labels", "List of overlap label names", "DMPlexDistribute", ovLabelNames, &numOvLabels, &flg));
3481   if (!flg) numOvLabels = 0;
3482   if (numOvLabels) {
3483     ((DM_Plex*) dm->data)->numOvLabels = numOvLabels;
3484     for (l = 0; l < numOvLabels; ++l) {
3485       PetscCall(DMGetLabel(dm, ovLabelNames[l], &((DM_Plex*) dm->data)->ovLabels[l]));
3486       PetscCheck(((DM_Plex*) dm->data)->ovLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovLabelNames[l]);
3487       PetscCall(PetscFree(ovLabelNames[l]));
3488     }
3489     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_values", "List of overlap label values", "DMPlexDistribute", ((DM_Plex*) dm->data)->ovValues, &numOvValues, &flg));
3490     if (!flg) numOvValues = 0;
3491     PetscCheck(numOvLabels == numOvValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvLabels, numOvValues);
3492 
3493     PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_exclude_labels", "List of overlap exclude label names", "DMPlexDistribute", ovExLabelNames, &numOvExLabels, &flg));
3494     if (!flg) numOvExLabels = 0;
3495     ((DM_Plex*) dm->data)->numOvExLabels = numOvExLabels;
3496     for (l = 0; l < numOvExLabels; ++l) {
3497       PetscCall(DMGetLabel(dm, ovExLabelNames[l], &((DM_Plex*) dm->data)->ovExLabels[l]));
3498       PetscCheck(((DM_Plex*) dm->data)->ovExLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovExLabelNames[l]);
3499       PetscCall(PetscFree(ovExLabelNames[l]));
3500     }
3501     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_exclude_values", "List of overlap exclude label values", "DMPlexDistribute", ((DM_Plex*) dm->data)->ovExValues, &numOvExValues, &flg));
3502     if (!flg) numOvExValues = 0;
3503     PetscCheck(numOvExLabels == numOvExValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of exclude labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvExLabels, numOvExValues);
3504   }
3505   PetscFunctionReturn(0);
3506 }
3507 
3508 static PetscErrorCode DMSetFromOptions_Plex(PetscOptionItems *PetscOptionsObject,DM dm)
3509 {
3510   PetscFunctionList ordlist;
3511   char                      oname[256];
3512   PetscReal                 volume = -1.0;
3513   PetscInt                  prerefine = 0, refine = 0, r, coarsen = 0, overlap = 0, extLayers = 0, dim;
3514   PetscBool                 uniformOrig, created = PETSC_FALSE, uniform = PETSC_TRUE, distribute, interpolate = PETSC_TRUE, coordSpace = PETSC_TRUE, remap = PETSC_TRUE, ghostCells = PETSC_FALSE, isHierarchy, ignoreModel = PETSC_FALSE, flg;
3515   DMPlexReorderDefaultFlag  reorder;
3516 
3517   PetscFunctionBegin;
3518   PetscValidHeaderSpecific(dm, DM_CLASSID, 2);
3519   PetscOptionsHeadBegin(PetscOptionsObject,"DMPlex Options");
3520   /* Handle automatic creation */
3521   PetscCall(DMGetDimension(dm, &dim));
3522   if (dim < 0) {
3523     PetscCall(DMPlexCreateFromOptions_Internal(PetscOptionsObject, &coordSpace, dm));
3524     created = PETSC_TRUE;
3525   }
3526   PetscCall(DMGetDimension(dm, &dim));
3527   /* Handle interpolation before distribution */
3528   PetscCall(PetscOptionsBool("-dm_plex_interpolate_pre", "Flag to interpolate mesh before distribution", "", interpolate, &interpolate, &flg));
3529   if (flg) {
3530     DMPlexInterpolatedFlag interpolated;
3531 
3532     PetscCall(DMPlexIsInterpolated(dm, &interpolated));
3533     if (interpolated == DMPLEX_INTERPOLATED_FULL && !interpolate) {
3534       DM udm;
3535 
3536       PetscCall(DMPlexUninterpolate(dm, &udm));
3537       PetscCall(DMPlexReplace_Static(dm, &udm));
3538     } else if (interpolated != DMPLEX_INTERPOLATED_FULL && interpolate) {
3539       DM idm;
3540 
3541       PetscCall(DMPlexInterpolate(dm, &idm));
3542       PetscCall(DMPlexReplace_Static(dm, &idm));
3543     }
3544   }
3545   /* Handle DMPlex refinement before distribution */
3546   PetscCall(PetscOptionsBool("-dm_refine_ignore_model", "Flag to ignore the geometry model when refining", "DMCreate", ignoreModel, &ignoreModel, &flg));
3547   if (flg) {((DM_Plex *) dm->data)->ignoreModel = ignoreModel;}
3548   PetscCall(DMPlexGetRefinementUniform(dm, &uniformOrig));
3549   PetscCall(PetscOptionsBoundedInt("-dm_refine_pre", "The number of refinements before distribution", "DMCreate", prerefine, &prerefine, NULL,0));
3550   PetscCall(PetscOptionsBool("-dm_refine_remap_pre", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
3551   PetscCall(PetscOptionsBool("-dm_refine_uniform_pre", "Flag for uniform refinement before distribution", "DMCreate", uniform, &uniform, &flg));
3552   if (flg) PetscCall(DMPlexSetRefinementUniform(dm, uniform));
3553   PetscCall(PetscOptionsReal("-dm_refine_volume_limit_pre", "The maximum cell volume after refinement before distribution", "DMCreate", volume, &volume, &flg));
3554   if (flg) {
3555     PetscCall(DMPlexSetRefinementUniform(dm, PETSC_FALSE));
3556     PetscCall(DMPlexSetRefinementLimit(dm, volume));
3557     prerefine = PetscMax(prerefine, 1);
3558   }
3559   for (r = 0; r < prerefine; ++r) {
3560     DM             rdm;
3561     PetscPointFunc coordFunc = ((DM_Plex*) dm->data)->coordFunc;
3562 
3563     PetscCall(DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm));
3564     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject) dm), &rdm));
3565     PetscCall(DMPlexReplace_Static(dm, &rdm));
3566     PetscCall(DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm));
3567     if (coordFunc && remap) {
3568       PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
3569       ((DM_Plex*) dm->data)->coordFunc = coordFunc;
3570     }
3571   }
3572   PetscCall(DMPlexSetRefinementUniform(dm, uniformOrig));
3573   /* Handle DMPlex extrusion before distribution */
3574   PetscCall(PetscOptionsBoundedInt("-dm_extrude", "The number of layers to extrude", "", extLayers, &extLayers, NULL, 0));
3575   if (extLayers) {
3576     DM edm;
3577 
3578     PetscCall(DMExtrude(dm, extLayers, &edm));
3579     PetscCall(DMPlexReplace_Static(dm, &edm));
3580     ((DM_Plex *) dm->data)->coordFunc = NULL;
3581     PetscCall(DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm));
3582     extLayers = 0;
3583   }
3584   /* Handle DMPlex reordering before distribution */
3585   PetscCall(DMPlexReorderGetDefault(dm, &reorder));
3586   PetscCall(MatGetOrderingList(&ordlist));
3587   PetscCall(PetscStrncpy(oname, MATORDERINGNATURAL, sizeof(oname)));
3588   PetscCall(PetscOptionsFList("-dm_plex_reorder", "Set mesh reordering type", "DMPlexGetOrdering", ordlist, MATORDERINGNATURAL, oname, sizeof(oname), &flg));
3589   if (reorder == DMPLEX_REORDER_DEFAULT_TRUE || flg) {
3590     DM pdm;
3591     IS perm;
3592 
3593     PetscCall(DMPlexGetOrdering(dm, oname, NULL, &perm));
3594     PetscCall(DMPlexPermute(dm, perm, &pdm));
3595     PetscCall(ISDestroy(&perm));
3596     PetscCall(DMPlexReplace_Static(dm, &pdm));
3597     PetscCall(DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm));
3598   }
3599   /* Handle DMPlex distribution */
3600   PetscCall(DMPlexDistributeGetDefault(dm, &distribute));
3601   PetscCall(PetscOptionsBool("-dm_distribute", "Flag to redistribute a mesh among processes", "DMPlexDistribute", distribute, &distribute, NULL));
3602   PetscCall(DMSetFromOptions_Overlap_Plex(PetscOptionsObject, dm, &overlap));
3603   if (distribute) {
3604     DM               pdm = NULL;
3605     PetscPartitioner part;
3606 
3607     PetscCall(DMPlexGetPartitioner(dm, &part));
3608     PetscCall(PetscPartitionerSetFromOptions(part));
3609     PetscCall(DMPlexDistribute(dm, overlap, NULL, &pdm));
3610     if (pdm) {
3611       PetscCall(DMPlexReplace_Static(dm, &pdm));
3612     }
3613   }
3614   /* Create coordinate space */
3615   if (created) {
3616     DM_Plex  *mesh = (DM_Plex *) dm->data;
3617     PetscInt  degree = 1;
3618     PetscBool flg;
3619 
3620     PetscCall(PetscOptionsBool("-dm_coord_space", "Use an FEM space for coordinates", "", coordSpace, &coordSpace, &flg));
3621     PetscCall(PetscOptionsInt("-dm_coord_petscspace_degree", "FEM degree for coordinate space", "", degree, &degree, NULL));
3622     if (coordSpace) PetscCall(DMPlexCreateCoordinateSpace(dm, degree, mesh->coordFunc));
3623     if (flg && !coordSpace) {
3624       DM           cdm;
3625       PetscDS      cds;
3626       PetscObject  obj;
3627       PetscClassId id;
3628 
3629       PetscCall(DMGetCoordinateDM(dm, &cdm));
3630       PetscCall(DMGetDS(cdm, &cds));
3631       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
3632       PetscCall(PetscObjectGetClassId(obj, &id));
3633       if (id == PETSCFE_CLASSID) {
3634         PetscContainer dummy;
3635 
3636         PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &dummy));
3637         PetscCall(PetscObjectSetName((PetscObject) dummy, "coordinates"));
3638         PetscCall(DMSetField(cdm, 0, NULL, (PetscObject) dummy));
3639         PetscCall(PetscContainerDestroy(&dummy));
3640         PetscCall(DMClearDS(cdm));
3641       }
3642       mesh->coordFunc = NULL;
3643     }
3644     PetscCall(PetscOptionsBool("-dm_sparse_localize", "Localize only necessary cells", "", dm->sparseLocalize, &dm->sparseLocalize, &flg));
3645     PetscCall(DMLocalizeCoordinates(dm));
3646   }
3647   /* Handle DMPlex refinement */
3648   remap = PETSC_TRUE;
3649   PetscCall(PetscOptionsBoundedInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL,0));
3650   PetscCall(PetscOptionsBool("-dm_refine_remap", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
3651   PetscCall(PetscOptionsBoundedInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy,0));
3652   if (refine) PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3653   if (refine && isHierarchy) {
3654     DM *dms, coarseDM;
3655 
3656     PetscCall(DMGetCoarseDM(dm, &coarseDM));
3657     PetscCall(PetscObjectReference((PetscObject)coarseDM));
3658     PetscCall(PetscMalloc1(refine,&dms));
3659     PetscCall(DMRefineHierarchy(dm, refine, dms));
3660     /* Total hack since we do not pass in a pointer */
3661     PetscCall(DMPlexSwap_Static(dm, dms[refine-1]));
3662     if (refine == 1) {
3663       PetscCall(DMSetCoarseDM(dm, dms[0]));
3664       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
3665     } else {
3666       PetscCall(DMSetCoarseDM(dm, dms[refine-2]));
3667       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
3668       PetscCall(DMSetCoarseDM(dms[0], dms[refine-1]));
3669       PetscCall(DMPlexSetRegularRefinement(dms[0], PETSC_TRUE));
3670     }
3671     PetscCall(DMSetCoarseDM(dms[refine-1], coarseDM));
3672     PetscCall(PetscObjectDereference((PetscObject)coarseDM));
3673     /* Free DMs */
3674     for (r = 0; r < refine; ++r) {
3675       PetscCall(DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dms[r]));
3676       PetscCall(DMDestroy(&dms[r]));
3677     }
3678     PetscCall(PetscFree(dms));
3679   } else {
3680     for (r = 0; r < refine; ++r) {
3681       DM             rdm;
3682       PetscPointFunc coordFunc = ((DM_Plex*) dm->data)->coordFunc;
3683 
3684       PetscCall(DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm));
3685       PetscCall(DMRefine(dm, PetscObjectComm((PetscObject) dm), &rdm));
3686       /* Total hack since we do not pass in a pointer */
3687       PetscCall(DMPlexReplace_Static(dm, &rdm));
3688       PetscCall(DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm));
3689       if (coordFunc && remap) {
3690         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
3691         ((DM_Plex*) dm->data)->coordFunc = coordFunc;
3692       }
3693     }
3694   }
3695   /* Handle DMPlex coarsening */
3696   PetscCall(PetscOptionsBoundedInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL,0));
3697   PetscCall(PetscOptionsBoundedInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy,0));
3698   if (coarsen && isHierarchy) {
3699     DM *dms;
3700 
3701     PetscCall(PetscMalloc1(coarsen, &dms));
3702     PetscCall(DMCoarsenHierarchy(dm, coarsen, dms));
3703     /* Free DMs */
3704     for (r = 0; r < coarsen; ++r) {
3705       PetscCall(DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dms[r]));
3706       PetscCall(DMDestroy(&dms[r]));
3707     }
3708     PetscCall(PetscFree(dms));
3709   } else {
3710     for (r = 0; r < coarsen; ++r) {
3711       DM             cdm;
3712       PetscPointFunc coordFunc = ((DM_Plex*) dm->data)->coordFunc;
3713 
3714       PetscCall(DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm));
3715       PetscCall(DMCoarsen(dm, PetscObjectComm((PetscObject) dm), &cdm));
3716       /* Total hack since we do not pass in a pointer */
3717       PetscCall(DMPlexReplace_Static(dm, &cdm));
3718       PetscCall(DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm));
3719       if (coordFunc) {
3720         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
3721         ((DM_Plex*) dm->data)->coordFunc = coordFunc;
3722       }
3723     }
3724   }
3725   /* Handle ghost cells */
3726   PetscCall(PetscOptionsBool("-dm_plex_create_fv_ghost_cells", "Flag to create finite volume ghost cells on the boundary", "DMCreate", ghostCells, &ghostCells, NULL));
3727   if (ghostCells) {
3728     DM   gdm;
3729     char lname[PETSC_MAX_PATH_LEN];
3730 
3731     lname[0] = '\0';
3732     PetscCall(PetscOptionsString("-dm_plex_fv_ghost_cells_label", "Label name for ghost cells boundary", "DMCreate", lname, lname, sizeof(lname), &flg));
3733     PetscCall(DMPlexConstructGhostCells(dm, flg ? lname : NULL, NULL, &gdm));
3734     PetscCall(DMPlexReplace_Static(dm, &gdm));
3735   }
3736   /* Handle 1D order */
3737   if (reorder != DMPLEX_REORDER_DEFAULT_FALSE && dim == 1) {
3738     DM           cdm, rdm;
3739     PetscDS      cds;
3740     PetscObject  obj;
3741     PetscClassId id = PETSC_OBJECT_CLASSID;
3742     IS           perm;
3743     PetscInt     Nf;
3744     PetscBool    distributed;
3745 
3746     PetscCall(DMPlexIsDistributed(dm, &distributed));
3747     PetscCall(DMGetCoordinateDM(dm, &cdm));
3748     PetscCall(DMGetDS(cdm, &cds));
3749     PetscCall(PetscDSGetNumFields(cds, &Nf));
3750     if (Nf) {
3751       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
3752       PetscCall(PetscObjectGetClassId(obj, &id));
3753     }
3754     if (!distributed && id != PETSCFE_CLASSID) {
3755       PetscCall(DMPlexGetOrdering1D(dm, &perm));
3756       PetscCall(DMPlexPermute(dm, perm, &rdm));
3757       PetscCall(DMPlexReplace_Static(dm, &rdm));
3758       PetscCall(ISDestroy(&perm));
3759     }
3760   }
3761   /* Handle */
3762   PetscCall(DMSetFromOptions_NonRefinement_Plex(PetscOptionsObject, dm));
3763   PetscOptionsHeadEnd();
3764   PetscFunctionReturn(0);
3765 }
3766 
3767 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm,Vec *vec)
3768 {
3769   PetscFunctionBegin;
3770   PetscCall(DMCreateGlobalVector_Section_Private(dm,vec));
3771   /* PetscCall(VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM)); */
3772   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void)) VecView_Plex));
3773   PetscCall(VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void)) VecView_Plex_Native));
3774   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void)) VecLoad_Plex));
3775   PetscCall(VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void)) VecLoad_Plex_Native));
3776   PetscFunctionReturn(0);
3777 }
3778 
3779 static PetscErrorCode DMCreateLocalVector_Plex(DM dm,Vec *vec)
3780 {
3781   PetscFunctionBegin;
3782   PetscCall(DMCreateLocalVector_Section_Private(dm,vec));
3783   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void)) VecView_Plex_Local));
3784   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void)) VecLoad_Plex_Local));
3785   PetscFunctionReturn(0);
3786 }
3787 
3788 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
3789 {
3790   PetscInt       depth, d;
3791 
3792   PetscFunctionBegin;
3793   PetscCall(DMPlexGetDepth(dm, &depth));
3794   if (depth == 1) {
3795     PetscCall(DMGetDimension(dm, &d));
3796     if (dim == 0)      PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
3797     else if (dim == d) PetscCall(DMPlexGetDepthStratum(dm, 1, pStart, pEnd));
3798     else               {*pStart = 0; *pEnd = 0;}
3799   } else {
3800     PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
3801   }
3802   PetscFunctionReturn(0);
3803 }
3804 
3805 static PetscErrorCode DMGetNeighbors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
3806 {
3807   PetscSF           sf;
3808   PetscInt          niranks, njranks, n;
3809   const PetscMPIInt *iranks, *jranks;
3810   DM_Plex           *data = (DM_Plex*) dm->data;
3811 
3812   PetscFunctionBegin;
3813   PetscCall(DMGetPointSF(dm, &sf));
3814   if (!data->neighbors) {
3815     PetscCall(PetscSFSetUp(sf));
3816     PetscCall(PetscSFGetRootRanks(sf, &njranks, &jranks, NULL, NULL, NULL));
3817     PetscCall(PetscSFGetLeafRanks(sf, &niranks, &iranks, NULL, NULL));
3818     PetscCall(PetscMalloc1(njranks + niranks + 1, &data->neighbors));
3819     PetscCall(PetscArraycpy(data->neighbors + 1, jranks, njranks));
3820     PetscCall(PetscArraycpy(data->neighbors + njranks + 1, iranks, niranks));
3821     n = njranks + niranks;
3822     PetscCall(PetscSortRemoveDupsMPIInt(&n, data->neighbors + 1));
3823     /* The following cast should never fail: can't have more neighbors than PETSC_MPI_INT_MAX */
3824     PetscCall(PetscMPIIntCast(n, data->neighbors));
3825   }
3826   if (nranks) *nranks = data->neighbors[0];
3827   if (ranks) {
3828     if (data->neighbors[0]) *ranks = data->neighbors + 1;
3829     else                    *ranks = NULL;
3830   }
3831   PetscFunctionReturn(0);
3832 }
3833 
3834 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM, DM, Mat, Vec, Vec);
3835 
3836 static PetscErrorCode DMInitialize_Plex(DM dm)
3837 {
3838   PetscFunctionBegin;
3839   dm->ops->view                            = DMView_Plex;
3840   dm->ops->load                            = DMLoad_Plex;
3841   dm->ops->setfromoptions                  = DMSetFromOptions_Plex;
3842   dm->ops->clone                           = DMClone_Plex;
3843   dm->ops->setup                           = DMSetUp_Plex;
3844   dm->ops->createlocalsection              = DMCreateLocalSection_Plex;
3845   dm->ops->createdefaultconstraints        = DMCreateDefaultConstraints_Plex;
3846   dm->ops->createglobalvector              = DMCreateGlobalVector_Plex;
3847   dm->ops->createlocalvector               = DMCreateLocalVector_Plex;
3848   dm->ops->getlocaltoglobalmapping         = NULL;
3849   dm->ops->createfieldis                   = NULL;
3850   dm->ops->createcoordinatedm              = DMCreateCoordinateDM_Plex;
3851   dm->ops->createcoordinatefield           = DMCreateCoordinateField_Plex;
3852   dm->ops->getcoloring                     = NULL;
3853   dm->ops->creatematrix                    = DMCreateMatrix_Plex;
3854   dm->ops->createinterpolation             = DMCreateInterpolation_Plex;
3855   dm->ops->createmassmatrix                = DMCreateMassMatrix_Plex;
3856   dm->ops->createmassmatrixlumped          = DMCreateMassMatrixLumped_Plex;
3857   dm->ops->createinjection                 = DMCreateInjection_Plex;
3858   dm->ops->refine                          = DMRefine_Plex;
3859   dm->ops->coarsen                         = DMCoarsen_Plex;
3860   dm->ops->refinehierarchy                 = DMRefineHierarchy_Plex;
3861   dm->ops->coarsenhierarchy                = DMCoarsenHierarchy_Plex;
3862   dm->ops->extrude                         = DMExtrude_Plex;
3863   dm->ops->globaltolocalbegin              = NULL;
3864   dm->ops->globaltolocalend                = NULL;
3865   dm->ops->localtoglobalbegin              = NULL;
3866   dm->ops->localtoglobalend                = NULL;
3867   dm->ops->destroy                         = DMDestroy_Plex;
3868   dm->ops->createsubdm                     = DMCreateSubDM_Plex;
3869   dm->ops->createsuperdm                   = DMCreateSuperDM_Plex;
3870   dm->ops->getdimpoints                    = DMGetDimPoints_Plex;
3871   dm->ops->locatepoints                    = DMLocatePoints_Plex;
3872   dm->ops->projectfunctionlocal            = DMProjectFunctionLocal_Plex;
3873   dm->ops->projectfunctionlabellocal       = DMProjectFunctionLabelLocal_Plex;
3874   dm->ops->projectfieldlocal               = DMProjectFieldLocal_Plex;
3875   dm->ops->projectfieldlabellocal          = DMProjectFieldLabelLocal_Plex;
3876   dm->ops->projectbdfieldlabellocal        = DMProjectBdFieldLabelLocal_Plex;
3877   dm->ops->computel2diff                   = DMComputeL2Diff_Plex;
3878   dm->ops->computel2gradientdiff           = DMComputeL2GradientDiff_Plex;
3879   dm->ops->computel2fielddiff              = DMComputeL2FieldDiff_Plex;
3880   dm->ops->getneighbors                    = DMGetNeighbors_Plex;
3881   PetscCall(PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertBoundaryValues_C",DMPlexInsertBoundaryValues_Plex));
3882   PetscCall(PetscObjectComposeFunction((PetscObject)dm,"DMPlexInsertTimeDerviativeBoundaryValues_C",DMPlexInsertTimeDerivativeBoundaryValues_Plex));
3883   PetscCall(PetscObjectComposeFunction((PetscObject)dm,"DMSetUpGLVisViewer_C",DMSetUpGLVisViewer_Plex));
3884   PetscCall(PetscObjectComposeFunction((PetscObject)dm,"DMCreateNeumannOverlap_C",DMCreateNeumannOverlap_Plex));
3885   PetscCall(PetscObjectComposeFunction((PetscObject)dm,"DMPlexGetOverlap_C",DMPlexGetOverlap_Plex));
3886   PetscCall(PetscObjectComposeFunction((PetscObject)dm,"DMPlexDistributeGetDefault_C",DMPlexDistributeGetDefault_Plex));
3887   PetscCall(PetscObjectComposeFunction((PetscObject)dm,"DMPlexDistributeSetDefault_C",DMPlexDistributeSetDefault_Plex));
3888   PetscCall(PetscObjectComposeFunction((PetscObject)dm,"DMPlexReorderGetDefault_C",DMPlexReorderGetDefault_Plex));
3889   PetscCall(PetscObjectComposeFunction((PetscObject)dm,"DMPlexReorderSetDefault_C",DMPlexReorderSetDefault_Plex));
3890   PetscCall(PetscObjectComposeFunction((PetscObject)dm,"DMInterpolateSolution_C",DMInterpolateSolution_Plex));
3891   PetscCall(PetscObjectComposeFunction((PetscObject)dm,"DMPlexGetOverlap_C",DMPlexGetOverlap_Plex));
3892   PetscCall(PetscObjectComposeFunction((PetscObject)dm,"DMPlexSetOverlap_C",DMPlexSetOverlap_Plex));
3893   PetscFunctionReturn(0);
3894 }
3895 
3896 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
3897 {
3898   DM_Plex        *mesh = (DM_Plex *) dm->data;
3899 
3900   PetscFunctionBegin;
3901   mesh->refct++;
3902   (*newdm)->data = mesh;
3903   PetscCall(PetscObjectChangeTypeName((PetscObject) *newdm, DMPLEX));
3904   PetscCall(DMInitialize_Plex(*newdm));
3905   PetscFunctionReturn(0);
3906 }
3907 
3908 /*MC
3909   DMPLEX = "plex" - A DM object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
3910                     In the local representation, Vecs contain all unknowns in the interior and shared boundary. This is
3911                     specified by a PetscSection object. Ownership in the global representation is determined by
3912                     ownership of the underlying DMPlex points. This is specified by another PetscSection object.
3913 
3914   Options Database Keys:
3915 + -dm_refine_pre                     - Refine mesh before distribution
3916 + -dm_refine_uniform_pre             - Choose uniform or generator-based refinement
3917 + -dm_refine_volume_limit_pre        - Cell volume limit after pre-refinement using generator
3918 . -dm_distribute                     - Distribute mesh across processes
3919 . -dm_distribute_overlap             - Number of cells to overlap for distribution
3920 . -dm_refine                         - Refine mesh after distribution
3921 . -dm_plex_hash_location             - Use grid hashing for point location
3922 . -dm_plex_hash_box_faces <n,m,p>    - The number of divisions in each direction of the grid hash
3923 . -dm_plex_partition_balance         - Attempt to evenly divide points on partition boundary between processes
3924 . -dm_plex_remesh_bd                 - Allow changes to the boundary on remeshing
3925 . -dm_plex_max_projection_height     - Maxmimum mesh point height used to project locally
3926 . -dm_plex_regular_refinement        - Use special nested projection algorithm for regular refinement
3927 . -dm_plex_check_all                 - Perform all shecks below
3928 . -dm_plex_check_symmetry            - Check that the adjacency information in the mesh is symmetric
3929 . -dm_plex_check_skeleton <celltype> - Check that each cell has the correct number of vertices
3930 . -dm_plex_check_faces <celltype>    - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
3931 . -dm_plex_check_geometry            - Check that cells have positive volume
3932 . -dm_view :mesh.tex:ascii_latex     - View the mesh in LaTeX/TikZ
3933 . -dm_plex_view_scale <num>          - Scale the TikZ
3934 - -dm_plex_print_fem <num>           - View FEM assembly information, such as element vectors and matrices
3935 
3936   Level: intermediate
3937 
3938 .seealso: `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`
3939 M*/
3940 
3941 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
3942 {
3943   DM_Plex       *mesh;
3944   PetscInt       unit;
3945 
3946   PetscFunctionBegin;
3947   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3948   PetscCall(PetscNewLog(dm,&mesh));
3949   dm->data = mesh;
3950 
3951   mesh->refct             = 1;
3952   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection));
3953   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection));
3954   mesh->refinementUniform = PETSC_TRUE;
3955   mesh->refinementLimit   = -1.0;
3956   mesh->distDefault       = PETSC_TRUE;
3957   mesh->reorderDefault    = DMPLEX_REORDER_DEFAULT_NOTSET;
3958   mesh->interpolated      = DMPLEX_INTERPOLATED_INVALID;
3959   mesh->interpolatedCollective = DMPLEX_INTERPOLATED_INVALID;
3960 
3961   PetscCall(PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner));
3962   mesh->remeshBd     = PETSC_FALSE;
3963 
3964   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
3965 
3966   mesh->depthState          = -1;
3967   mesh->celltypeState       = -1;
3968   mesh->printTol            = 1.0e-10;
3969 
3970   PetscCall(DMInitialize_Plex(dm));
3971   PetscFunctionReturn(0);
3972 }
3973 
3974 /*@
3975   DMPlexCreate - Creates a DMPlex object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
3976 
3977   Collective
3978 
3979   Input Parameter:
3980 . comm - The communicator for the DMPlex object
3981 
3982   Output Parameter:
3983 . mesh  - The DMPlex object
3984 
3985   Level: beginner
3986 
3987 @*/
3988 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
3989 {
3990   PetscFunctionBegin;
3991   PetscValidPointer(mesh,2);
3992   PetscCall(DMCreate(comm, mesh));
3993   PetscCall(DMSetType(*mesh, DMPLEX));
3994   PetscFunctionReturn(0);
3995 }
3996 
3997 /*@C
3998   DMPlexBuildFromCellListParallel - Build distributed DMPLEX topology from a list of vertices for each cell (common mesh generator output)
3999 
4000   Input Parameters:
4001 + dm - The DM
4002 . numCells - The number of cells owned by this process
4003 . numVertices - The number of vertices to be owned by this process, or PETSC_DECIDE
4004 . NVertices - The global number of vertices, or PETSC_DETERMINE
4005 . numCorners - The number of vertices for each cell
4006 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4007 
4008   Output Parameters:
4009 + vertexSF - (Optional) SF describing complete vertex ownership
4010 - verticesAdjSaved - (Optional) vertex adjacency array
4011 
4012   Notes:
4013   Two triangles sharing a face
4014 $
4015 $        2
4016 $      / | \
4017 $     /  |  \
4018 $    /   |   \
4019 $   0  0 | 1  3
4020 $    \   |   /
4021 $     \  |  /
4022 $      \ | /
4023 $        1
4024 would have input
4025 $  numCells = 2, numVertices = 4
4026 $  cells = [0 1 2  1 3 2]
4027 $
4028 which would result in the DMPlex
4029 $
4030 $        4
4031 $      / | \
4032 $     /  |  \
4033 $    /   |   \
4034 $   2  0 | 1  5
4035 $    \   |   /
4036 $     \  |  /
4037 $      \ | /
4038 $        3
4039 
4040   Vertices are implicitly numbered consecutively 0,...,NVertices.
4041   Each rank owns a chunk of numVertices consecutive vertices.
4042   If numVertices is PETSC_DECIDE, PETSc will distribute them as evenly as possible using PetscLayout.
4043   If NVertices is PETSC_DETERMINE and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
4044   If only NVertices is PETSC_DETERMINE, it is computed as the sum of numVertices over all ranks.
4045 
4046   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
4047 
4048   Not currently supported in Fortran.
4049 
4050   Level: advanced
4051 
4052 .seealso: `DMPlexBuildFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildCoordinatesFromCellListParallel()`
4053 @*/
4054 PetscErrorCode DMPlexBuildFromCellListParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, const PetscInt cells[], PetscSF *vertexSF, PetscInt **verticesAdjSaved)
4055 {
4056   PetscSF         sfPoint;
4057   PetscLayout     layout;
4058   PetscInt        numVerticesAdj, *verticesAdj, *cones, c, p;
4059 
4060   PetscFunctionBegin;
4061   PetscValidLogicalCollectiveInt(dm,NVertices,4);
4062   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList,dm,0,0,0));
4063   /* Get/check global number of vertices */
4064   {
4065     PetscInt NVerticesInCells, i;
4066     const PetscInt len = numCells * numCorners;
4067 
4068     /* NVerticesInCells = max(cells) + 1 */
4069     NVerticesInCells = PETSC_MIN_INT;
4070     for (i=0; i<len; i++) if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4071     ++NVerticesInCells;
4072     PetscCallMPI(MPI_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject) dm)));
4073 
4074     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
4075     else PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT,NVertices,NVerticesInCells);
4076   }
4077   /* Count locally unique vertices */
4078   {
4079     PetscHSetI vhash;
4080     PetscInt off = 0;
4081 
4082     PetscCall(PetscHSetICreate(&vhash));
4083     for (c = 0; c < numCells; ++c) {
4084       for (p = 0; p < numCorners; ++p) {
4085         PetscCall(PetscHSetIAdd(vhash, cells[c*numCorners+p]));
4086       }
4087     }
4088     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
4089     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
4090     else { verticesAdj = *verticesAdjSaved; }
4091     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
4092     PetscCall(PetscHSetIDestroy(&vhash));
4093     PetscCheck(off == numVerticesAdj,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
4094   }
4095   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
4096   /* Create cones */
4097   PetscCall(DMPlexSetChart(dm, 0, numCells+numVerticesAdj));
4098   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
4099   PetscCall(DMSetUp(dm));
4100   PetscCall(DMPlexGetCones(dm,&cones));
4101   for (c = 0; c < numCells; ++c) {
4102     for (p = 0; p < numCorners; ++p) {
4103       const PetscInt gv = cells[c*numCorners+p];
4104       PetscInt       lv;
4105 
4106       /* Positions within verticesAdj form 0-based local vertex numbering;
4107          we need to shift it by numCells to get correct DAG points (cells go first) */
4108       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
4109       PetscCheck(lv >= 0,PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
4110       cones[c*numCorners+p] = lv+numCells;
4111     }
4112   }
4113   /* Build point sf */
4114   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
4115   PetscCall(PetscLayoutSetSize(layout, NVertices));
4116   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
4117   PetscCall(PetscLayoutSetBlockSize(layout, 1));
4118   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
4119   PetscCall(PetscLayoutDestroy(&layout));
4120   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
4121   PetscCall(PetscObjectSetName((PetscObject) sfPoint, "point SF"));
4122   if (dm->sf) {
4123     const char *prefix;
4124 
4125     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
4126     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
4127   }
4128   PetscCall(DMSetPointSF(dm, sfPoint));
4129   PetscCall(PetscSFDestroy(&sfPoint));
4130   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)(*vertexSF), "Vertex Ownership SF"));
4131   /* Fill in the rest of the topology structure */
4132   PetscCall(DMPlexSymmetrize(dm));
4133   PetscCall(DMPlexStratify(dm));
4134   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList,dm,0,0,0));
4135   PetscFunctionReturn(0);
4136 }
4137 
4138 /*@C
4139   DMPlexBuildCoordinatesFromCellListParallel - Build DM coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4140 
4141   Input Parameters:
4142 + dm - The DM
4143 . spaceDim - The spatial dimension used for coordinates
4144 . sfVert - SF describing complete vertex ownership
4145 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4146 
4147   Level: advanced
4148 
4149   Notes:
4150   Not currently supported in Fortran.
4151 
4152 .seealso: `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellListParallel()`
4153 @*/
4154 PetscErrorCode DMPlexBuildCoordinatesFromCellListParallel(DM dm, PetscInt spaceDim, PetscSF sfVert, const PetscReal vertexCoords[])
4155 {
4156   PetscSection   coordSection;
4157   Vec            coordinates;
4158   PetscScalar   *coords;
4159   PetscInt       numVertices, numVerticesAdj, coordSize, v, vStart, vEnd;
4160 
4161   PetscFunctionBegin;
4162   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList,dm,0,0,0));
4163   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
4164   PetscCheck(vStart >= 0 && vEnd >= 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4165   PetscCall(DMSetCoordinateDim(dm, spaceDim));
4166   PetscCall(PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL));
4167   PetscCheck(vEnd - vStart == numVerticesAdj,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Supplied sfVert has wrong number of leaves = %" PetscInt_FMT " != %" PetscInt_FMT " = vEnd - vStart",numVerticesAdj,vEnd - vStart);
4168   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4169   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4170   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
4171   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
4172   for (v = vStart; v < vEnd; ++v) {
4173     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
4174     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
4175   }
4176   PetscCall(PetscSectionSetUp(coordSection));
4177   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4178   PetscCall(VecCreate(PetscObjectComm((PetscObject)dm), &coordinates));
4179   PetscCall(VecSetBlockSize(coordinates, spaceDim));
4180   PetscCall(PetscObjectSetName((PetscObject) coordinates, "coordinates"));
4181   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4182   PetscCall(VecSetType(coordinates,VECSTANDARD));
4183   PetscCall(VecGetArray(coordinates, &coords));
4184   {
4185     MPI_Datatype coordtype;
4186 
4187     /* Need a temp buffer for coords if we have complex/single */
4188     PetscCallMPI(MPI_Type_contiguous(spaceDim, MPIU_SCALAR, &coordtype));
4189     PetscCallMPI(MPI_Type_commit(&coordtype));
4190 #if defined(PETSC_USE_COMPLEX)
4191     {
4192     PetscScalar *svertexCoords;
4193     PetscInt    i;
4194     PetscCall(PetscMalloc1(numVertices*spaceDim,&svertexCoords));
4195     for (i=0; i<numVertices*spaceDim; i++) svertexCoords[i] = vertexCoords[i];
4196     PetscCall(PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords,MPI_REPLACE));
4197     PetscCall(PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords,MPI_REPLACE));
4198     PetscCall(PetscFree(svertexCoords));
4199     }
4200 #else
4201     PetscCall(PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords,MPI_REPLACE));
4202     PetscCall(PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords,MPI_REPLACE));
4203 #endif
4204     PetscCallMPI(MPI_Type_free(&coordtype));
4205   }
4206   PetscCall(VecRestoreArray(coordinates, &coords));
4207   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4208   PetscCall(VecDestroy(&coordinates));
4209   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList,dm,0,0,0));
4210   PetscFunctionReturn(0);
4211 }
4212 
4213 /*@
4214   DMPlexCreateFromCellListParallelPetsc - Create distributed DMPLEX from a list of vertices for each cell (common mesh generator output)
4215 
4216   Input Parameters:
4217 + comm - The communicator
4218 . dim - The topological dimension of the mesh
4219 . numCells - The number of cells owned by this process
4220 . numVertices - The number of vertices owned by this process, or PETSC_DECIDE
4221 . NVertices - The global number of vertices, or PETSC_DECIDE
4222 . numCorners - The number of vertices for each cell
4223 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4224 . cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4225 . spaceDim - The spatial dimension used for coordinates
4226 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4227 
4228   Output Parameters:
4229 + dm - The DM
4230 . vertexSF - (Optional) SF describing complete vertex ownership
4231 - verticesAdjSaved - (Optional) vertex adjacency array
4232 
4233   Notes:
4234   This function is just a convenient sequence of DMCreate(), DMSetType(), DMSetDimension(),
4235   DMPlexBuildFromCellListParallel(), DMPlexInterpolate(), DMPlexBuildCoordinatesFromCellListParallel()
4236 
4237   See DMPlexBuildFromCellListParallel() for an example and details about the topology-related parameters.
4238   See DMPlexBuildCoordinatesFromCellListParallel() for details about the geometry-related parameters.
4239 
4240   Level: intermediate
4241 
4242 .seealso: `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
4243 @*/
4244 PetscErrorCode DMPlexCreateFromCellListParallelPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, PetscInt **verticesAdj, DM *dm)
4245 {
4246   PetscSF        sfVert;
4247 
4248   PetscFunctionBegin;
4249   PetscCall(DMCreate(comm, dm));
4250   PetscCall(DMSetType(*dm, DMPLEX));
4251   PetscValidLogicalCollectiveInt(*dm, dim, 2);
4252   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
4253   PetscCall(DMSetDimension(*dm, dim));
4254   PetscCall(DMPlexBuildFromCellListParallel(*dm, numCells, numVertices, NVertices, numCorners, cells, &sfVert, verticesAdj));
4255   if (interpolate) {
4256     DM idm;
4257 
4258     PetscCall(DMPlexInterpolate(*dm, &idm));
4259     PetscCall(DMDestroy(dm));
4260     *dm  = idm;
4261   }
4262   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
4263   if (vertexSF) *vertexSF = sfVert;
4264   else PetscCall(PetscSFDestroy(&sfVert));
4265   PetscFunctionReturn(0);
4266 }
4267 
4268 /*@C
4269   DMPlexBuildFromCellList - Build DMPLEX topology from a list of vertices for each cell (common mesh generator output)
4270 
4271   Input Parameters:
4272 + dm - The DM
4273 . numCells - The number of cells owned by this process
4274 . numVertices - The number of vertices owned by this process, or PETSC_DETERMINE
4275 . numCorners - The number of vertices for each cell
4276 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4277 
4278   Level: advanced
4279 
4280   Notes:
4281   Two triangles sharing a face
4282 $
4283 $        2
4284 $      / | \
4285 $     /  |  \
4286 $    /   |   \
4287 $   0  0 | 1  3
4288 $    \   |   /
4289 $     \  |  /
4290 $      \ | /
4291 $        1
4292 would have input
4293 $  numCells = 2, numVertices = 4
4294 $  cells = [0 1 2  1 3 2]
4295 $
4296 which would result in the DMPlex
4297 $
4298 $        4
4299 $      / | \
4300 $     /  |  \
4301 $    /   |   \
4302 $   2  0 | 1  5
4303 $    \   |   /
4304 $     \  |  /
4305 $      \ | /
4306 $        3
4307 
4308   If numVertices is PETSC_DETERMINE, it is computed by PETSc as the maximum vertex index in cells + 1.
4309 
4310   Not currently supported in Fortran.
4311 
4312 .seealso: `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListPetsc()`
4313 @*/
4314 PetscErrorCode DMPlexBuildFromCellList(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const PetscInt cells[])
4315 {
4316   PetscInt      *cones, c, p, dim;
4317 
4318   PetscFunctionBegin;
4319   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList,dm,0,0,0));
4320   PetscCall(DMGetDimension(dm, &dim));
4321   /* Get/check global number of vertices */
4322   {
4323     PetscInt NVerticesInCells, i;
4324     const PetscInt len = numCells * numCorners;
4325 
4326     /* NVerticesInCells = max(cells) + 1 */
4327     NVerticesInCells = PETSC_MIN_INT;
4328     for (i=0; i<len; i++) if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4329     ++NVerticesInCells;
4330 
4331     if (numVertices == PETSC_DECIDE) numVertices = NVerticesInCells;
4332     else PetscCheck(numVertices >= NVerticesInCells,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Specified number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT,numVertices,NVerticesInCells);
4333   }
4334   PetscCall(DMPlexSetChart(dm, 0, numCells+numVertices));
4335   for (c = 0; c < numCells; ++c) {
4336     PetscCall(DMPlexSetConeSize(dm, c, numCorners));
4337   }
4338   PetscCall(DMSetUp(dm));
4339   PetscCall(DMPlexGetCones(dm,&cones));
4340   for (c = 0; c < numCells; ++c) {
4341     for (p = 0; p < numCorners; ++p) {
4342       cones[c*numCorners+p] = cells[c*numCorners+p]+numCells;
4343     }
4344   }
4345   PetscCall(DMPlexSymmetrize(dm));
4346   PetscCall(DMPlexStratify(dm));
4347   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList,dm,0,0,0));
4348   PetscFunctionReturn(0);
4349 }
4350 
4351 /*@C
4352   DMPlexBuildCoordinatesFromCellList - Build DM coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4353 
4354   Input Parameters:
4355 + dm - The DM
4356 . spaceDim - The spatial dimension used for coordinates
4357 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4358 
4359   Level: advanced
4360 
4361   Notes:
4362   Not currently supported in Fortran.
4363 
4364 .seealso: `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellList()`
4365 @*/
4366 PetscErrorCode DMPlexBuildCoordinatesFromCellList(DM dm, PetscInt spaceDim, const PetscReal vertexCoords[])
4367 {
4368   PetscSection   coordSection;
4369   Vec            coordinates;
4370   DM             cdm;
4371   PetscScalar   *coords;
4372   PetscInt       v, vStart, vEnd, d;
4373 
4374   PetscFunctionBegin;
4375   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList,dm,0,0,0));
4376   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
4377   PetscCheck(vStart >= 0 && vEnd >= 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4378   PetscCall(DMSetCoordinateDim(dm, spaceDim));
4379   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4380   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4381   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
4382   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
4383   for (v = vStart; v < vEnd; ++v) {
4384     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
4385     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
4386   }
4387   PetscCall(PetscSectionSetUp(coordSection));
4388 
4389   PetscCall(DMGetCoordinateDM(dm, &cdm));
4390   PetscCall(DMCreateLocalVector(cdm, &coordinates));
4391   PetscCall(VecSetBlockSize(coordinates, spaceDim));
4392   PetscCall(PetscObjectSetName((PetscObject) coordinates, "coordinates"));
4393   PetscCall(VecGetArrayWrite(coordinates, &coords));
4394   for (v = 0; v < vEnd-vStart; ++v) {
4395     for (d = 0; d < spaceDim; ++d) {
4396       coords[v*spaceDim+d] = vertexCoords[v*spaceDim+d];
4397     }
4398   }
4399   PetscCall(VecRestoreArrayWrite(coordinates, &coords));
4400   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4401   PetscCall(VecDestroy(&coordinates));
4402   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList,dm,0,0,0));
4403   PetscFunctionReturn(0);
4404 }
4405 
4406 /*@
4407   DMPlexCreateFromCellListPetsc - Create DMPLEX from a list of vertices for each cell (common mesh generator output), but only process 0 takes in the input
4408 
4409   Collective on comm
4410 
4411   Input Parameters:
4412 + comm - The communicator
4413 . dim - The topological dimension of the mesh
4414 . numCells - The number of cells, only on process 0
4415 . numVertices - The number of vertices owned by this process, or PETSC_DECIDE, only on process 0
4416 . numCorners - The number of vertices for each cell, only on process 0
4417 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4418 . cells - An array of numCells*numCorners numbers, the vertices for each cell, only on process 0
4419 . spaceDim - The spatial dimension used for coordinates
4420 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex, only on process 0
4421 
4422   Output Parameter:
4423 . dm - The DM, which only has points on process 0
4424 
4425   Notes:
4426   This function is just a convenient sequence of DMCreate(), DMSetType(), DMSetDimension(), DMPlexBuildFromCellList(),
4427   DMPlexInterpolate(), DMPlexBuildCoordinatesFromCellList()
4428 
4429   See DMPlexBuildFromCellList() for an example and details about the topology-related parameters.
4430   See DMPlexBuildCoordinatesFromCellList() for details about the geometry-related parameters.
4431   See DMPlexCreateFromCellListParallelPetsc() for parallel input
4432 
4433   Level: intermediate
4434 
4435 .seealso: `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellList()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
4436 @*/
4437 PetscErrorCode DMPlexCreateFromCellListPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], DM *dm)
4438 {
4439   PetscMPIInt    rank;
4440 
4441   PetscFunctionBegin;
4442   PetscCheck(dim,comm, PETSC_ERR_ARG_OUTOFRANGE, "This is not appropriate for 0-dimensional meshes. Consider either creating the DM using DMPlexCreateFromDAG(), by hand, or using DMSwarm.");
4443   PetscCallMPI(MPI_Comm_rank(comm, &rank));
4444   PetscCall(DMCreate(comm, dm));
4445   PetscCall(DMSetType(*dm, DMPLEX));
4446   PetscCall(DMSetDimension(*dm, dim));
4447   if (rank == 0) PetscCall(DMPlexBuildFromCellList(*dm, numCells, numVertices, numCorners, cells));
4448   else           PetscCall(DMPlexBuildFromCellList(*dm, 0, 0, 0, NULL));
4449   if (interpolate) {
4450     DM idm;
4451 
4452     PetscCall(DMPlexInterpolate(*dm, &idm));
4453     PetscCall(DMDestroy(dm));
4454     *dm  = idm;
4455   }
4456   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, vertexCoords));
4457   else           PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, NULL));
4458   PetscFunctionReturn(0);
4459 }
4460 
4461 /*@
4462   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a DM
4463 
4464   Input Parameters:
4465 + dm - The empty DM object, usually from DMCreate() and DMSetDimension()
4466 . depth - The depth of the DAG
4467 . numPoints - Array of size depth + 1 containing the number of points at each depth
4468 . coneSize - The cone size of each point
4469 . cones - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
4470 . coneOrientations - The orientation of each cone point
4471 - vertexCoords - An array of numPoints[0]*spacedim numbers representing the coordinates of each vertex, with spacedim the value set via DMSetCoordinateDim()
4472 
4473   Output Parameter:
4474 . dm - The DM
4475 
4476   Note: Two triangles sharing a face would have input
4477 $  depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
4478 $  cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
4479 $ vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
4480 $
4481 which would result in the DMPlex
4482 $
4483 $        4
4484 $      / | \
4485 $     /  |  \
4486 $    /   |   \
4487 $   2  0 | 1  5
4488 $    \   |   /
4489 $     \  |  /
4490 $      \ | /
4491 $        3
4492 $
4493 $ Notice that all points are numbered consecutively, unlike DMPlexCreateFromCellListPetsc()
4494 
4495   Level: advanced
4496 
4497 .seealso: `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
4498 @*/
4499 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
4500 {
4501   Vec            coordinates;
4502   PetscSection   coordSection;
4503   PetscScalar    *coords;
4504   PetscInt       coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
4505 
4506   PetscFunctionBegin;
4507   PetscCall(DMGetDimension(dm, &dim));
4508   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
4509   PetscCheck(dimEmbed >= dim,PETSC_COMM_SELF,PETSC_ERR_PLIB,"Embedding dimension %" PetscInt_FMT " cannot be less than intrinsic dimension %" PetscInt_FMT,dimEmbed,dim);
4510   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
4511   PetscCall(DMPlexSetChart(dm, pStart, pEnd));
4512   for (p = pStart; p < pEnd; ++p) {
4513     PetscCall(DMPlexSetConeSize(dm, p, coneSize[p-pStart]));
4514     if (firstVertex < 0 && !coneSize[p - pStart]) {
4515       firstVertex = p - pStart;
4516     }
4517   }
4518   PetscCheck(firstVertex >= 0 || !numPoints[0],PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Expected %" PetscInt_FMT " vertices but could not find any", numPoints[0]);
4519   PetscCall(DMSetUp(dm)); /* Allocate space for cones */
4520   for (p = pStart, off = 0; p < pEnd; off += coneSize[p-pStart], ++p) {
4521     PetscCall(DMPlexSetCone(dm, p, &cones[off]));
4522     PetscCall(DMPlexSetConeOrientation(dm, p, &coneOrientations[off]));
4523   }
4524   PetscCall(DMPlexSymmetrize(dm));
4525   PetscCall(DMPlexStratify(dm));
4526   /* Build coordinates */
4527   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4528   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4529   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dimEmbed));
4530   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex+numPoints[0]));
4531   for (v = firstVertex; v < firstVertex+numPoints[0]; ++v) {
4532     PetscCall(PetscSectionSetDof(coordSection, v, dimEmbed));
4533     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed));
4534   }
4535   PetscCall(PetscSectionSetUp(coordSection));
4536   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4537   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
4538   PetscCall(PetscObjectSetName((PetscObject) coordinates, "coordinates"));
4539   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4540   PetscCall(VecSetBlockSize(coordinates, dimEmbed));
4541   PetscCall(VecSetType(coordinates,VECSTANDARD));
4542   if (vertexCoords) {
4543     PetscCall(VecGetArray(coordinates, &coords));
4544     for (v = 0; v < numPoints[0]; ++v) {
4545       PetscInt off;
4546 
4547       PetscCall(PetscSectionGetOffset(coordSection, v+firstVertex, &off));
4548       for (d = 0; d < dimEmbed; ++d) {
4549         coords[off+d] = vertexCoords[v*dimEmbed+d];
4550       }
4551     }
4552   }
4553   PetscCall(VecRestoreArray(coordinates, &coords));
4554   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4555   PetscCall(VecDestroy(&coordinates));
4556   PetscFunctionReturn(0);
4557 }
4558 
4559 /*@C
4560   DMPlexCreateCellVertexFromFile - Create a DMPlex mesh from a simple cell-vertex file.
4561 
4562 + comm        - The MPI communicator
4563 . filename    - Name of the .dat file
4564 - interpolate - Create faces and edges in the mesh
4565 
4566   Output Parameter:
4567 . dm  - The DM object representing the mesh
4568 
4569   Note: The format is the simplest possible:
4570 $ Ne
4571 $ v0 v1 ... vk
4572 $ Nv
4573 $ x y z marker
4574 
4575   Level: beginner
4576 
4577 .seealso: `DMPlexCreateFromFile()`, `DMPlexCreateMedFromFile()`, `DMPlexCreateGmsh()`, `DMPlexCreate()`
4578 @*/
4579 PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
4580 {
4581   DMLabel         marker;
4582   PetscViewer     viewer;
4583   Vec             coordinates;
4584   PetscSection    coordSection;
4585   PetscScalar    *coords;
4586   char            line[PETSC_MAX_PATH_LEN];
4587   PetscInt        dim = 3, cdim = 3, coordSize, v, c, d;
4588   PetscMPIInt     rank;
4589   int             snum, Nv, Nc, Ncn, Nl;
4590 
4591   PetscFunctionBegin;
4592   PetscCallMPI(MPI_Comm_rank(comm, &rank));
4593   PetscCall(PetscViewerCreate(comm, &viewer));
4594   PetscCall(PetscViewerSetType(viewer, PETSCVIEWERASCII));
4595   PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
4596   PetscCall(PetscViewerFileSetName(viewer, filename));
4597   if (rank == 0) {
4598     PetscCall(PetscViewerRead(viewer, line, 4, NULL, PETSC_STRING));
4599     snum = sscanf(line, "%d %d %d %d", &Nc, &Nv, &Ncn, &Nl);
4600     PetscCheck(snum == 4,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4601   } else {
4602     Nc = Nv = Ncn = Nl = 0;
4603   }
4604   PetscCall(DMCreate(comm, dm));
4605   PetscCall(DMSetType(*dm, DMPLEX));
4606   PetscCall(DMPlexSetChart(*dm, 0, Nc+Nv));
4607   PetscCall(DMSetDimension(*dm, dim));
4608   PetscCall(DMSetCoordinateDim(*dm, cdim));
4609   /* Read topology */
4610   if (rank == 0) {
4611     char     format[PETSC_MAX_PATH_LEN];
4612     PetscInt cone[8];
4613     int      vbuf[8], v;
4614 
4615     for (c = 0; c < Ncn; ++c) {format[c*3+0] = '%'; format[c*3+1] = 'd'; format[c*3+2] = ' ';}
4616     format[Ncn*3-1] = '\0';
4617     for (c = 0; c < Nc; ++c) PetscCall(DMPlexSetConeSize(*dm, c, Ncn));
4618     PetscCall(DMSetUp(*dm));
4619     for (c = 0; c < Nc; ++c) {
4620       PetscCall(PetscViewerRead(viewer, line, Ncn, NULL, PETSC_STRING));
4621       switch (Ncn) {
4622         case 2: snum = sscanf(line, format, &vbuf[0], &vbuf[1]);break;
4623         case 3: snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2]);break;
4624         case 4: snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3]);break;
4625         case 6: snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5]);break;
4626         case 8: snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);break;
4627         default: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No cell shape with %d vertices", Ncn);
4628       }
4629       PetscCheck(snum == Ncn,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4630       for (v = 0; v < Ncn; ++v) cone[v] = vbuf[v] + Nc;
4631       /* Hexahedra are inverted */
4632       if (Ncn == 8) {
4633         PetscInt tmp = cone[1];
4634         cone[1] = cone[3];
4635         cone[3] = tmp;
4636       }
4637       PetscCall(DMPlexSetCone(*dm, c, cone));
4638     }
4639   }
4640   PetscCall(DMPlexSymmetrize(*dm));
4641   PetscCall(DMPlexStratify(*dm));
4642   /* Read coordinates */
4643   PetscCall(DMGetCoordinateSection(*dm, &coordSection));
4644   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4645   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
4646   PetscCall(PetscSectionSetChart(coordSection, Nc, Nc + Nv));
4647   for (v = Nc; v < Nc+Nv; ++v) {
4648     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
4649     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
4650   }
4651   PetscCall(PetscSectionSetUp(coordSection));
4652   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4653   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
4654   PetscCall(PetscObjectSetName((PetscObject) coordinates, "coordinates"));
4655   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4656   PetscCall(VecSetBlockSize(coordinates, cdim));
4657   PetscCall(VecSetType(coordinates, VECSTANDARD));
4658   PetscCall(VecGetArray(coordinates, &coords));
4659   if (rank == 0) {
4660     char   format[PETSC_MAX_PATH_LEN];
4661     double x[3];
4662     int    l, val[3];
4663 
4664     if (Nl) {
4665       for (l = 0; l < Nl; ++l) {format[l*3+0] = '%'; format[l*3+1] = 'd'; format[l*3+2] = ' ';}
4666       format[Nl*3-1] = '\0';
4667       PetscCall(DMCreateLabel(*dm, "marker"));
4668       PetscCall(DMGetLabel(*dm, "marker", &marker));
4669     }
4670     for (v = 0; v < Nv; ++v) {
4671       PetscCall(PetscViewerRead(viewer, line, 3+Nl, NULL, PETSC_STRING));
4672       snum = sscanf(line, "%lg %lg %lg", &x[0], &x[1], &x[2]);
4673       PetscCheck(snum == 3,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4674       switch (Nl) {
4675         case 0: snum = 0;break;
4676         case 1: snum = sscanf(line, format, &val[0]);break;
4677         case 2: snum = sscanf(line, format, &val[0], &val[1]);break;
4678         case 3: snum = sscanf(line, format, &val[0], &val[1], &val[2]);break;
4679         default: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Request support for %d labels", Nl);
4680       }
4681       PetscCheck(snum == Nl,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4682       for (d = 0; d < cdim; ++d) coords[v*cdim+d] = x[d];
4683       for (l = 0; l < Nl; ++l) PetscCall(DMLabelSetValue(marker, v+Nc, val[l]));
4684     }
4685   }
4686   PetscCall(VecRestoreArray(coordinates, &coords));
4687   PetscCall(DMSetCoordinatesLocal(*dm, coordinates));
4688   PetscCall(VecDestroy(&coordinates));
4689   PetscCall(PetscViewerDestroy(&viewer));
4690   if (interpolate) {
4691     DM      idm;
4692     DMLabel bdlabel;
4693 
4694     PetscCall(DMPlexInterpolate(*dm, &idm));
4695     PetscCall(DMDestroy(dm));
4696     *dm  = idm;
4697 
4698     if (!Nl) {
4699       PetscCall(DMCreateLabel(*dm, "marker"));
4700       PetscCall(DMGetLabel(*dm, "marker", &bdlabel));
4701       PetscCall(DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel));
4702       PetscCall(DMPlexLabelComplete(*dm, bdlabel));
4703     }
4704   }
4705   PetscFunctionReturn(0);
4706 }
4707 
4708 /*@C
4709   DMPlexCreateFromFile - This takes a filename and produces a DM
4710 
4711   Input Parameters:
4712 + comm - The communicator
4713 . filename - A file name
4714 . plexname - The object name of the resulting DM, also used for intra-datafile lookup by some formats
4715 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
4716 
4717   Output Parameter:
4718 . dm - The DM
4719 
4720   Options Database Keys:
4721 . -dm_plex_create_from_hdf5_xdmf - use the PETSC_VIEWER_HDF5_XDMF format for reading HDF5
4722 
4723   Use -dm_plex_create_ prefix to pass options to the internal PetscViewer, e.g.
4724 $ -dm_plex_create_viewer_hdf5_collective
4725 
4726   Notes:
4727   Using PETSCVIEWERHDF5 type with PETSC_VIEWER_HDF5_PETSC format, one can save multiple DMPlex
4728   meshes in a single HDF5 file. This in turn requires one to name the DMPlex object with PetscObjectSetName()
4729   before saving it with DMView() and before loading it with DMLoad() for identification of the mesh object.
4730   The input parameter name is thus used to name the DMPlex object when DMPlexCreateFromFile() internally
4731   calls DMLoad(). Currently, name is ignored for other viewer types and/or formats.
4732 
4733   Level: beginner
4734 
4735 .seealso: `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`, `PetscObjectSetName()`, `DMView()`, `DMLoad()`
4736 @*/
4737 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], const char plexname[], PetscBool interpolate, DM *dm)
4738 {
4739   const char  extGmsh[]      = ".msh";
4740   const char  extGmsh2[]     = ".msh2";
4741   const char  extGmsh4[]     = ".msh4";
4742   const char  extCGNS[]      = ".cgns";
4743   const char  extExodus[]    = ".exo";
4744   const char  extExodus_e[]  = ".e";
4745   const char  extGenesis[]   = ".gen";
4746   const char  extFluent[]    = ".cas";
4747   const char  extHDF5[]      = ".h5";
4748   const char  extMed[]       = ".med";
4749   const char  extPLY[]       = ".ply";
4750   const char  extEGADSLite[] = ".egadslite";
4751   const char  extEGADS[]     = ".egads";
4752   const char  extIGES[]      = ".igs";
4753   const char  extSTEP[]      = ".stp";
4754   const char  extCV[]        = ".dat";
4755   size_t      len;
4756   PetscBool   isGmsh, isGmsh2, isGmsh4, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isMed, isPLY, isEGADSLite, isEGADS, isIGES, isSTEP, isCV;
4757   PetscMPIInt rank;
4758 
4759   PetscFunctionBegin;
4760   PetscValidCharPointer(filename, 2);
4761   if (plexname) PetscValidCharPointer(plexname, 3);
4762   PetscValidPointer(dm, 5);
4763   PetscCall(DMInitializePackage());
4764   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromFile,0,0,0,0));
4765   PetscCallMPI(MPI_Comm_rank(comm, &rank));
4766   PetscCall(PetscStrlen(filename, &len));
4767   PetscCheck(len,comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
4768 
4769 #define CheckExtension(extension__,is_extension__) do {                                        \
4770     PetscAssert(sizeof(extension__), comm, PETSC_ERR_PLIB, "Zero-size extension: %s", extension__); \
4771     /* don't count the null-terminator at the end */                                           \
4772     const size_t ext_len = sizeof(extension__)-1;                                              \
4773     if (len < ext_len) {                                                                       \
4774       is_extension__ = PETSC_FALSE;                                                            \
4775     } else {                                                                                   \
4776       PetscCall(PetscStrncmp(filename+len-ext_len, extension__, ext_len, &is_extension__));    \
4777     }                                                                                          \
4778   } while (0)
4779 
4780   CheckExtension(extGmsh, isGmsh);
4781   CheckExtension(extGmsh2, isGmsh2);
4782   CheckExtension(extGmsh4, isGmsh4);
4783   CheckExtension(extCGNS, isCGNS);
4784   CheckExtension(extExodus, isExodus);
4785   if (!isExodus) CheckExtension(extExodus_e, isExodus);
4786   CheckExtension(extGenesis, isGenesis);
4787   CheckExtension(extFluent, isFluent);
4788   CheckExtension(extHDF5, isHDF5);
4789   CheckExtension(extMed, isMed);
4790   CheckExtension(extPLY, isPLY);
4791   CheckExtension(extEGADSLite, isEGADSLite);
4792   CheckExtension(extEGADS, isEGADS);
4793   CheckExtension(extIGES, isIGES);
4794   CheckExtension(extSTEP, isSTEP);
4795   CheckExtension(extCV, isCV);
4796 
4797 #undef CheckExtension
4798 
4799   if (isGmsh || isGmsh2 || isGmsh4) {
4800     PetscCall(DMPlexCreateGmshFromFile(comm, filename, interpolate, dm));
4801   } else if (isCGNS) {
4802     PetscCall(DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm));
4803   } else if (isExodus || isGenesis) {
4804     PetscCall(DMPlexCreateExodusFromFile(comm, filename, interpolate, dm));
4805   } else if (isFluent) {
4806     PetscCall(DMPlexCreateFluentFromFile(comm, filename, interpolate, dm));
4807   } else if (isHDF5) {
4808     PetscBool      load_hdf5_xdmf = PETSC_FALSE;
4809     PetscViewer viewer;
4810 
4811     /* PETSC_VIEWER_HDF5_XDMF is used if the filename ends with .xdmf.h5, or if -dm_plex_create_from_hdf5_xdmf option is present */
4812     PetscCall(PetscStrncmp(&filename[PetscMax(0,len-8)], ".xdmf",  5, &load_hdf5_xdmf));
4813     PetscCall(PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &load_hdf5_xdmf, NULL));
4814     PetscCall(PetscViewerCreate(comm, &viewer));
4815     PetscCall(PetscViewerSetType(viewer, PETSCVIEWERHDF5));
4816     PetscCall(PetscViewerSetOptionsPrefix(viewer, "dm_plex_create_"));
4817     PetscCall(PetscViewerSetFromOptions(viewer));
4818     PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
4819     PetscCall(PetscViewerFileSetName(viewer, filename));
4820 
4821     PetscCall(DMCreate(comm, dm));
4822     PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
4823     PetscCall(DMSetType(*dm, DMPLEX));
4824     if (load_hdf5_xdmf) PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF));
4825     PetscCall(DMLoad(*dm, viewer));
4826     if (load_hdf5_xdmf) PetscCall(PetscViewerPopFormat(viewer));
4827     PetscCall(PetscViewerDestroy(&viewer));
4828 
4829     if (interpolate) {
4830       DM idm;
4831 
4832       PetscCall(DMPlexInterpolate(*dm, &idm));
4833       PetscCall(DMDestroy(dm));
4834       *dm  = idm;
4835     }
4836   } else if (isMed) {
4837     PetscCall(DMPlexCreateMedFromFile(comm, filename, interpolate, dm));
4838   } else if (isPLY) {
4839     PetscCall(DMPlexCreatePLYFromFile(comm, filename, interpolate, dm));
4840   } else if (isEGADSLite || isEGADS || isIGES || isSTEP) {
4841     if (isEGADSLite) PetscCall(DMPlexCreateEGADSLiteFromFile(comm, filename, dm));
4842     else             PetscCall(DMPlexCreateEGADSFromFile(comm, filename, dm));
4843     if (!interpolate) {
4844       DM udm;
4845 
4846       PetscCall(DMPlexUninterpolate(*dm, &udm));
4847       PetscCall(DMDestroy(dm));
4848       *dm  = udm;
4849     }
4850   } else if (isCV) {
4851     PetscCall(DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm));
4852   } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
4853   PetscCall(PetscStrlen(plexname, &len));
4854   if (len) PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
4855   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromFile,0,0,0,0));
4856   PetscFunctionReturn(0);
4857 }
4858