xref: /petsc/src/dm/impls/plex/plexcreate.c (revision 46560f82ebbebbe6388a7cb77cc9fc314bc23602)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h> /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>   /*I   "petscdmplex.h"   I*/
4 #include <petscsf.h>
5 #include <petscdmplextransform.h>
6 #include <petscdmlabelephemeral.h>
7 #include <petsc/private/kernels/blockmatmult.h>
8 #include <petsc/private/kernels/blockinvert.h>
9 
10 PetscLogEvent DMPLEX_CreateFromFile, DMPLEX_BuildFromCellList, DMPLEX_BuildCoordinatesFromCellList;
11 
12 /* External function declarations here */
13 static PetscErrorCode DMInitialize_Plex(DM dm);
14 
15 /* This copies internal things in the Plex structure that we generally want when making a new, related Plex */
16 PetscErrorCode DMPlexCopy_Internal(DM dmin, PetscBool copyPeriodicity, PetscBool copyOverlap, DM dmout)
17 {
18   const PetscReal         *maxCell, *Lstart, *L;
19   PetscBool                dist;
20   DMPlexReorderDefaultFlag reorder;
21 
22   PetscFunctionBegin;
23   if (copyPeriodicity) {
24     PetscCall(DMGetPeriodicity(dmin, &maxCell, &Lstart, &L));
25     PetscCall(DMSetPeriodicity(dmout, maxCell, Lstart, L));
26   }
27   PetscCall(DMPlexDistributeGetDefault(dmin, &dist));
28   PetscCall(DMPlexDistributeSetDefault(dmout, dist));
29   PetscCall(DMPlexReorderGetDefault(dmin, &reorder));
30   PetscCall(DMPlexReorderSetDefault(dmout, reorder));
31   ((DM_Plex *)dmout->data)->useHashLocation = ((DM_Plex *)dmin->data)->useHashLocation;
32   if (copyOverlap) PetscCall(DMPlexSetOverlap_Plex(dmout, dmin, 0));
33   PetscFunctionReturn(0);
34 }
35 
36 /* Replace dm with the contents of ndm, and then destroy ndm
37    - Share the DM_Plex structure
38    - Share the coordinates
39    - Share the SF
40 */
41 PetscErrorCode DMPlexReplace_Internal(DM dm, DM *ndm)
42 {
43   PetscSF          sf;
44   DM               dmNew = *ndm, coordDM, coarseDM;
45   Vec              coords;
46   const PetscReal *maxCell, *Lstart, *L;
47   PetscInt         dim, cdim;
48 
49   PetscFunctionBegin;
50   if (dm == dmNew) {
51     PetscCall(DMDestroy(ndm));
52     PetscFunctionReturn(0);
53   }
54   dm->setupcalled = dmNew->setupcalled;
55   PetscCall(DMGetDimension(dmNew, &dim));
56   PetscCall(DMSetDimension(dm, dim));
57   PetscCall(DMGetCoordinateDim(dmNew, &cdim));
58   PetscCall(DMSetCoordinateDim(dm, cdim));
59   PetscCall(DMGetPointSF(dmNew, &sf));
60   PetscCall(DMSetPointSF(dm, sf));
61   PetscCall(DMGetCoordinateDM(dmNew, &coordDM));
62   PetscCall(DMGetCoordinatesLocal(dmNew, &coords));
63   PetscCall(DMSetCoordinateDM(dm, coordDM));
64   PetscCall(DMSetCoordinatesLocal(dm, coords));
65   PetscCall(DMGetCellCoordinateDM(dmNew, &coordDM));
66   PetscCall(DMGetCellCoordinatesLocal(dmNew, &coords));
67   PetscCall(DMSetCellCoordinateDM(dm, coordDM));
68   PetscCall(DMSetCellCoordinatesLocal(dm, coords));
69   /* Do not want to create the coordinate field if it does not already exist, so do not call DMGetCoordinateField() */
70   PetscCall(DMFieldDestroy(&dm->coordinates[0].field));
71   dm->coordinates[0].field            = dmNew->coordinates[0].field;
72   ((DM_Plex *)dmNew->data)->coordFunc = ((DM_Plex *)dm->data)->coordFunc;
73   PetscCall(DMGetPeriodicity(dmNew, &maxCell, &Lstart, &L));
74   PetscCall(DMSetPeriodicity(dm, maxCell, Lstart, L));
75   PetscCall(DMDestroy_Plex(dm));
76   PetscCall(DMInitialize_Plex(dm));
77   dm->data = dmNew->data;
78   ((DM_Plex *)dmNew->data)->refct++;
79   PetscCall(DMDestroyLabelLinkList_Internal(dm));
80   PetscCall(DMCopyLabels(dmNew, dm, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL));
81   PetscCall(DMGetCoarseDM(dmNew, &coarseDM));
82   PetscCall(DMSetCoarseDM(dm, coarseDM));
83   PetscCall(DMDestroy(ndm));
84   PetscFunctionReturn(0);
85 }
86 
87 /* Swap dm with the contents of dmNew
88    - Swap the DM_Plex structure
89    - Swap the coordinates
90    - Swap the point PetscSF
91 */
92 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
93 {
94   DM          coordDMA, coordDMB;
95   Vec         coordsA, coordsB;
96   PetscSF     sfA, sfB;
97   DMField     fieldTmp;
98   void       *tmp;
99   DMLabelLink listTmp;
100   DMLabel     depthTmp;
101   PetscInt    tmpI;
102 
103   PetscFunctionBegin;
104   if (dmA == dmB) PetscFunctionReturn(0);
105   PetscCall(DMGetPointSF(dmA, &sfA));
106   PetscCall(DMGetPointSF(dmB, &sfB));
107   PetscCall(PetscObjectReference((PetscObject)sfA));
108   PetscCall(DMSetPointSF(dmA, sfB));
109   PetscCall(DMSetPointSF(dmB, sfA));
110   PetscCall(PetscObjectDereference((PetscObject)sfA));
111 
112   PetscCall(DMGetCoordinateDM(dmA, &coordDMA));
113   PetscCall(DMGetCoordinateDM(dmB, &coordDMB));
114   PetscCall(PetscObjectReference((PetscObject)coordDMA));
115   PetscCall(DMSetCoordinateDM(dmA, coordDMB));
116   PetscCall(DMSetCoordinateDM(dmB, coordDMA));
117   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
118 
119   PetscCall(DMGetCoordinatesLocal(dmA, &coordsA));
120   PetscCall(DMGetCoordinatesLocal(dmB, &coordsB));
121   PetscCall(PetscObjectReference((PetscObject)coordsA));
122   PetscCall(DMSetCoordinatesLocal(dmA, coordsB));
123   PetscCall(DMSetCoordinatesLocal(dmB, coordsA));
124   PetscCall(PetscObjectDereference((PetscObject)coordsA));
125 
126   PetscCall(DMGetCellCoordinateDM(dmA, &coordDMA));
127   PetscCall(DMGetCellCoordinateDM(dmB, &coordDMB));
128   PetscCall(PetscObjectReference((PetscObject)coordDMA));
129   PetscCall(DMSetCellCoordinateDM(dmA, coordDMB));
130   PetscCall(DMSetCellCoordinateDM(dmB, coordDMA));
131   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
132 
133   PetscCall(DMGetCellCoordinatesLocal(dmA, &coordsA));
134   PetscCall(DMGetCellCoordinatesLocal(dmB, &coordsB));
135   PetscCall(PetscObjectReference((PetscObject)coordsA));
136   PetscCall(DMSetCellCoordinatesLocal(dmA, coordsB));
137   PetscCall(DMSetCellCoordinatesLocal(dmB, coordsA));
138   PetscCall(PetscObjectDereference((PetscObject)coordsA));
139 
140   fieldTmp                  = dmA->coordinates[0].field;
141   dmA->coordinates[0].field = dmB->coordinates[0].field;
142   dmB->coordinates[0].field = fieldTmp;
143   fieldTmp                  = dmA->coordinates[1].field;
144   dmA->coordinates[1].field = dmB->coordinates[1].field;
145   dmB->coordinates[1].field = fieldTmp;
146   tmp                       = dmA->data;
147   dmA->data                 = dmB->data;
148   dmB->data                 = tmp;
149   listTmp                   = dmA->labels;
150   dmA->labels               = dmB->labels;
151   dmB->labels               = listTmp;
152   depthTmp                  = dmA->depthLabel;
153   dmA->depthLabel           = dmB->depthLabel;
154   dmB->depthLabel           = depthTmp;
155   depthTmp                  = dmA->celltypeLabel;
156   dmA->celltypeLabel        = dmB->celltypeLabel;
157   dmB->celltypeLabel        = depthTmp;
158   tmpI                      = dmA->levelup;
159   dmA->levelup              = dmB->levelup;
160   dmB->levelup              = tmpI;
161   PetscFunctionReturn(0);
162 }
163 
164 static PetscErrorCode DMPlexInterpolateInPlace_Internal(DM dm)
165 {
166   DM idm;
167 
168   PetscFunctionBegin;
169   PetscCall(DMPlexInterpolate(dm, &idm));
170   PetscCall(DMPlexCopyCoordinates(dm, idm));
171   PetscCall(DMPlexReplace_Internal(dm, &idm));
172   PetscFunctionReturn(0);
173 }
174 
175 /*@C
176   DMPlexCreateCoordinateSpace - Creates a finite element space for the coordinates
177 
178   Collective on dm
179 
180   Input Parameters:
181 + DM        - The DM
182 . degree    - The degree of the finite element or PETSC_DECIDE
183 - coordFunc - An optional function to map new points from refinement to the surface
184 
185   Level: advanced
186 
187 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `PetscPointFunc`, `PetscFECreateLagrange()`, `DMGetCoordinateDM()`
188 @*/
189 PetscErrorCode DMPlexCreateCoordinateSpace(DM dm, PetscInt degree, PetscPointFunc coordFunc)
190 {
191   DM_Plex     *mesh = (DM_Plex *)dm->data;
192   DM           cdm;
193   PetscDS      cds;
194   PetscFE      fe;
195   PetscClassId id;
196 
197   PetscFunctionBegin;
198   PetscCall(DMGetCoordinateDM(dm, &cdm));
199   PetscCall(DMGetDS(cdm, &cds));
200   PetscCall(PetscDSGetDiscretization(cds, 0, (PetscObject *)&fe));
201   PetscCall(PetscObjectGetClassId((PetscObject)fe, &id));
202   if (id != PETSCFE_CLASSID) {
203     PetscInt dim, dE, qorder;
204 
205     PetscCall(DMGetDimension(dm, &dim));
206     PetscCall(DMGetCoordinateDim(dm, &dE));
207     qorder = degree;
208     PetscObjectOptionsBegin((PetscObject)cdm);
209     PetscCall(PetscOptionsBoundedInt("-default_quadrature_order", "Quadrature order is one less than quadrature points per edge", "DMPlexCreateCoordinateSpace", qorder, &qorder, NULL, 0));
210     PetscOptionsEnd();
211     if (degree == PETSC_DECIDE) fe = NULL;
212     else {
213       DMPolytopeType ct;
214       PetscInt       cStart, cEnd, ctTmp;
215 
216       PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
217       // Want to match cell types
218       if (cEnd > cStart) PetscCall(DMPlexGetCellType(dm, cStart, &ct));
219       else ct = DM_POLYTOPE_UNKNOWN;
220       ctTmp = (PetscInt)ct;
221       PetscCallMPI(MPI_Allreduce(MPI_IN_PLACE, &ctTmp, 1, MPIU_INT, MPI_MIN, PetscObjectComm((PetscObject)dm)));
222       ct = (DMPolytopeType)ctTmp;
223       // Work around current bug
224       if (ct == DM_POLYTOPE_SEG_PRISM_TENSOR || ct == DM_POLYTOPE_TRI_PRISM_TENSOR || ct == DM_POLYTOPE_QUAD_PRISM_TENSOR) fe = NULL;
225       else PetscCall(PetscFECreateLagrangeByCell(PETSC_COMM_SELF, dim, dE, ct, degree, qorder, &fe));
226     }
227     if (fe) PetscCall(DMProjectCoordinates(dm, fe));
228     PetscCall(PetscFEDestroy(&fe));
229   }
230   mesh->coordFunc = coordFunc;
231   PetscFunctionReturn(0);
232 }
233 
234 /*@
235   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
236 
237   Collective
238 
239   Input Parameters:
240 + comm - The communicator for the `DM` object
241 . dim - The spatial dimension
242 . simplex - Flag for simplicial cells, otherwise they are tensor product cells
243 . interpolate - Flag to create intermediate mesh pieces (edges, faces)
244 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
245 
246   Output Parameter:
247 . dm - The `DM` object
248 
249   Level: beginner
250 
251 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMSetType()`, `DMCreate()`
252 @*/
253 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscReal refinementLimit, DM *newdm)
254 {
255   DM          dm;
256   PetscMPIInt rank;
257 
258   PetscFunctionBegin;
259   PetscCall(DMCreate(comm, &dm));
260   PetscCall(DMSetType(dm, DMPLEX));
261   PetscCall(DMSetDimension(dm, dim));
262   PetscCallMPI(MPI_Comm_rank(comm, &rank));
263   switch (dim) {
264   case 2:
265     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "triangular"));
266     else PetscCall(PetscObjectSetName((PetscObject)dm, "quadrilateral"));
267     break;
268   case 3:
269     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "tetrahedral"));
270     else PetscCall(PetscObjectSetName((PetscObject)dm, "hexahedral"));
271     break;
272   default:
273     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
274   }
275   if (rank) {
276     PetscInt numPoints[2] = {0, 0};
277     PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL));
278   } else {
279     switch (dim) {
280     case 2:
281       if (simplex) {
282         PetscInt    numPoints[2]        = {4, 2};
283         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
284         PetscInt    cones[6]            = {2, 3, 4, 5, 4, 3};
285         PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
286         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
287 
288         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
289       } else {
290         PetscInt    numPoints[2]        = {6, 2};
291         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
292         PetscInt    cones[8]            = {2, 3, 4, 5, 3, 6, 7, 4};
293         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
294         PetscScalar vertexCoords[12]    = {-1.0, -0.5, 0.0, -0.5, 0.0, 0.5, -1.0, 0.5, 1.0, -0.5, 1.0, 0.5};
295 
296         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
297       }
298       break;
299     case 3:
300       if (simplex) {
301         PetscInt    numPoints[2]        = {5, 2};
302         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
303         PetscInt    cones[8]            = {4, 3, 5, 2, 5, 3, 4, 6};
304         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
305         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0};
306 
307         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
308       } else {
309         PetscInt    numPoints[2]         = {12, 2};
310         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
311         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9, 5, 4, 10, 11, 7, 12, 13, 8};
312         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
313         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5, -1.0, 0.5, -0.5, 0.0, 0.5, -0.5, 0.0, -0.5, -0.5, -1.0, -0.5, 0.5, 0.0, -0.5, 0.5, 0.0, 0.5, 0.5, -1.0, 0.5, 0.5, 1.0, 0.5, -0.5, 1.0, -0.5, -0.5, 1.0, -0.5, 0.5, 1.0, 0.5, 0.5};
314 
315         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
316       }
317       break;
318     default:
319       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
320     }
321   }
322   *newdm = dm;
323   if (refinementLimit > 0.0) {
324     DM          rdm;
325     const char *name;
326 
327     PetscCall(DMPlexSetRefinementUniform(*newdm, PETSC_FALSE));
328     PetscCall(DMPlexSetRefinementLimit(*newdm, refinementLimit));
329     PetscCall(DMRefine(*newdm, comm, &rdm));
330     PetscCall(PetscObjectGetName((PetscObject)*newdm, &name));
331     PetscCall(PetscObjectSetName((PetscObject)rdm, name));
332     PetscCall(DMDestroy(newdm));
333     *newdm = rdm;
334   }
335   if (interpolate) {
336     DM idm;
337 
338     PetscCall(DMPlexInterpolate(*newdm, &idm));
339     PetscCall(DMDestroy(newdm));
340     *newdm = idm;
341   }
342   PetscFunctionReturn(0);
343 }
344 
345 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
346 {
347   const PetscInt numVertices    = 2;
348   PetscInt       markerRight    = 1;
349   PetscInt       markerLeft     = 1;
350   PetscBool      markerSeparate = PETSC_FALSE;
351   Vec            coordinates;
352   PetscSection   coordSection;
353   PetscScalar   *coords;
354   PetscInt       coordSize;
355   PetscMPIInt    rank;
356   PetscInt       cdim = 1, v;
357 
358   PetscFunctionBegin;
359   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
360   if (markerSeparate) {
361     markerRight = 2;
362     markerLeft  = 1;
363   }
364   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
365   if (rank == 0) {
366     PetscCall(DMPlexSetChart(dm, 0, numVertices));
367     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
368     PetscCall(DMSetLabelValue(dm, "marker", 0, markerLeft));
369     PetscCall(DMSetLabelValue(dm, "marker", 1, markerRight));
370   }
371   PetscCall(DMPlexSymmetrize(dm));
372   PetscCall(DMPlexStratify(dm));
373   /* Build coordinates */
374   PetscCall(DMSetCoordinateDim(dm, cdim));
375   PetscCall(DMGetCoordinateSection(dm, &coordSection));
376   PetscCall(PetscSectionSetNumFields(coordSection, 1));
377   PetscCall(PetscSectionSetChart(coordSection, 0, numVertices));
378   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
379   for (v = 0; v < numVertices; ++v) {
380     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
381     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
382   }
383   PetscCall(PetscSectionSetUp(coordSection));
384   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
385   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
386   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
387   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
388   PetscCall(VecSetBlockSize(coordinates, cdim));
389   PetscCall(VecSetType(coordinates, VECSTANDARD));
390   PetscCall(VecGetArray(coordinates, &coords));
391   coords[0] = lower[0];
392   coords[1] = upper[0];
393   PetscCall(VecRestoreArray(coordinates, &coords));
394   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
395   PetscCall(VecDestroy(&coordinates));
396   PetscFunctionReturn(0);
397 }
398 
399 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
400 {
401   const PetscInt numVertices    = (edges[0] + 1) * (edges[1] + 1);
402   const PetscInt numEdges       = edges[0] * (edges[1] + 1) + (edges[0] + 1) * edges[1];
403   PetscInt       markerTop      = 1;
404   PetscInt       markerBottom   = 1;
405   PetscInt       markerRight    = 1;
406   PetscInt       markerLeft     = 1;
407   PetscBool      markerSeparate = PETSC_FALSE;
408   Vec            coordinates;
409   PetscSection   coordSection;
410   PetscScalar   *coords;
411   PetscInt       coordSize;
412   PetscMPIInt    rank;
413   PetscInt       v, vx, vy;
414 
415   PetscFunctionBegin;
416   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
417   if (markerSeparate) {
418     markerTop    = 3;
419     markerBottom = 1;
420     markerRight  = 2;
421     markerLeft   = 4;
422   }
423   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
424   if (rank == 0) {
425     PetscInt e, ex, ey;
426 
427     PetscCall(DMPlexSetChart(dm, 0, numEdges + numVertices));
428     for (e = 0; e < numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
429     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
430     for (vx = 0; vx <= edges[0]; vx++) {
431       for (ey = 0; ey < edges[1]; ey++) {
432         PetscInt edge   = vx * edges[1] + ey + edges[0] * (edges[1] + 1);
433         PetscInt vertex = ey * (edges[0] + 1) + vx + numEdges;
434         PetscInt cone[2];
435 
436         cone[0] = vertex;
437         cone[1] = vertex + edges[0] + 1;
438         PetscCall(DMPlexSetCone(dm, edge, cone));
439         if (vx == edges[0]) {
440           PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
441           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
442           if (ey == edges[1] - 1) {
443             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
444             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerRight));
445           }
446         } else if (vx == 0) {
447           PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
448           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
449           if (ey == edges[1] - 1) {
450             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
451             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft));
452           }
453         }
454       }
455     }
456     for (vy = 0; vy <= edges[1]; vy++) {
457       for (ex = 0; ex < edges[0]; ex++) {
458         PetscInt edge   = vy * edges[0] + ex;
459         PetscInt vertex = vy * (edges[0] + 1) + ex + numEdges;
460         PetscInt cone[2];
461 
462         cone[0] = vertex;
463         cone[1] = vertex + 1;
464         PetscCall(DMPlexSetCone(dm, edge, cone));
465         if (vy == edges[1]) {
466           PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
467           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
468           if (ex == edges[0] - 1) {
469             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
470             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerTop));
471           }
472         } else if (vy == 0) {
473           PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
474           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
475           if (ex == edges[0] - 1) {
476             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
477             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom));
478           }
479         }
480       }
481     }
482   }
483   PetscCall(DMPlexSymmetrize(dm));
484   PetscCall(DMPlexStratify(dm));
485   /* Build coordinates */
486   PetscCall(DMSetCoordinateDim(dm, 2));
487   PetscCall(DMGetCoordinateSection(dm, &coordSection));
488   PetscCall(PetscSectionSetNumFields(coordSection, 1));
489   PetscCall(PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices));
490   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 2));
491   for (v = numEdges; v < numEdges + numVertices; ++v) {
492     PetscCall(PetscSectionSetDof(coordSection, v, 2));
493     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 2));
494   }
495   PetscCall(PetscSectionSetUp(coordSection));
496   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
497   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
498   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
499   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
500   PetscCall(VecSetBlockSize(coordinates, 2));
501   PetscCall(VecSetType(coordinates, VECSTANDARD));
502   PetscCall(VecGetArray(coordinates, &coords));
503   for (vy = 0; vy <= edges[1]; ++vy) {
504     for (vx = 0; vx <= edges[0]; ++vx) {
505       coords[(vy * (edges[0] + 1) + vx) * 2 + 0] = lower[0] + ((upper[0] - lower[0]) / edges[0]) * vx;
506       coords[(vy * (edges[0] + 1) + vx) * 2 + 1] = lower[1] + ((upper[1] - lower[1]) / edges[1]) * vy;
507     }
508   }
509   PetscCall(VecRestoreArray(coordinates, &coords));
510   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
511   PetscCall(VecDestroy(&coordinates));
512   PetscFunctionReturn(0);
513 }
514 
515 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
516 {
517   PetscInt     vertices[3], numVertices;
518   PetscInt     numFaces       = 2 * faces[0] * faces[1] + 2 * faces[1] * faces[2] + 2 * faces[0] * faces[2];
519   PetscInt     markerTop      = 1;
520   PetscInt     markerBottom   = 1;
521   PetscInt     markerFront    = 1;
522   PetscInt     markerBack     = 1;
523   PetscInt     markerRight    = 1;
524   PetscInt     markerLeft     = 1;
525   PetscBool    markerSeparate = PETSC_FALSE;
526   Vec          coordinates;
527   PetscSection coordSection;
528   PetscScalar *coords;
529   PetscInt     coordSize;
530   PetscMPIInt  rank;
531   PetscInt     v, vx, vy, vz;
532   PetscInt     voffset, iface = 0, cone[4];
533 
534   PetscFunctionBegin;
535   PetscCheck(faces[0] >= 1 && faces[1] >= 1 && faces[2] >= 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
536   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
537   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
538   if (markerSeparate) {
539     markerBottom = 1;
540     markerTop    = 2;
541     markerFront  = 3;
542     markerBack   = 4;
543     markerRight  = 5;
544     markerLeft   = 6;
545   }
546   vertices[0] = faces[0] + 1;
547   vertices[1] = faces[1] + 1;
548   vertices[2] = faces[2] + 1;
549   numVertices = vertices[0] * vertices[1] * vertices[2];
550   if (rank == 0) {
551     PetscInt f;
552 
553     PetscCall(DMPlexSetChart(dm, 0, numFaces + numVertices));
554     for (f = 0; f < numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
555     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
556 
557     /* Side 0 (Top) */
558     for (vy = 0; vy < faces[1]; vy++) {
559       for (vx = 0; vx < faces[0]; vx++) {
560         voffset = numFaces + vertices[0] * vertices[1] * (vertices[2] - 1) + vy * vertices[0] + vx;
561         cone[0] = voffset;
562         cone[1] = voffset + 1;
563         cone[2] = voffset + vertices[0] + 1;
564         cone[3] = voffset + vertices[0];
565         PetscCall(DMPlexSetCone(dm, iface, cone));
566         PetscCall(DMSetLabelValue(dm, "marker", iface, markerTop));
567         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerTop));
568         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerTop));
569         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerTop));
570         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerTop));
571         iface++;
572       }
573     }
574 
575     /* Side 1 (Bottom) */
576     for (vy = 0; vy < faces[1]; vy++) {
577       for (vx = 0; vx < faces[0]; vx++) {
578         voffset = numFaces + vy * (faces[0] + 1) + vx;
579         cone[0] = voffset + 1;
580         cone[1] = voffset;
581         cone[2] = voffset + vertices[0];
582         cone[3] = voffset + vertices[0] + 1;
583         PetscCall(DMPlexSetCone(dm, iface, cone));
584         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBottom));
585         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBottom));
586         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBottom));
587         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerBottom));
588         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerBottom));
589         iface++;
590       }
591     }
592 
593     /* Side 2 (Front) */
594     for (vz = 0; vz < faces[2]; vz++) {
595       for (vx = 0; vx < faces[0]; vx++) {
596         voffset = numFaces + vz * vertices[0] * vertices[1] + vx;
597         cone[0] = voffset;
598         cone[1] = voffset + 1;
599         cone[2] = voffset + vertices[0] * vertices[1] + 1;
600         cone[3] = voffset + vertices[0] * vertices[1];
601         PetscCall(DMPlexSetCone(dm, iface, cone));
602         PetscCall(DMSetLabelValue(dm, "marker", iface, markerFront));
603         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerFront));
604         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerFront));
605         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerFront));
606         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerFront));
607         iface++;
608       }
609     }
610 
611     /* Side 3 (Back) */
612     for (vz = 0; vz < faces[2]; vz++) {
613       for (vx = 0; vx < faces[0]; vx++) {
614         voffset = numFaces + vz * vertices[0] * vertices[1] + vertices[0] * (vertices[1] - 1) + vx;
615         cone[0] = voffset + vertices[0] * vertices[1];
616         cone[1] = voffset + vertices[0] * vertices[1] + 1;
617         cone[2] = voffset + 1;
618         cone[3] = voffset;
619         PetscCall(DMPlexSetCone(dm, iface, cone));
620         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBack));
621         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBack));
622         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBack));
623         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerBack));
624         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerBack));
625         iface++;
626       }
627     }
628 
629     /* Side 4 (Left) */
630     for (vz = 0; vz < faces[2]; vz++) {
631       for (vy = 0; vy < faces[1]; vy++) {
632         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0];
633         cone[0] = voffset;
634         cone[1] = voffset + vertices[0] * vertices[1];
635         cone[2] = voffset + vertices[0] * vertices[1] + vertices[0];
636         cone[3] = voffset + vertices[0];
637         PetscCall(DMPlexSetCone(dm, iface, cone));
638         PetscCall(DMSetLabelValue(dm, "marker", iface, markerLeft));
639         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerLeft));
640         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerLeft));
641         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[1] + 0, markerLeft));
642         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerLeft));
643         iface++;
644       }
645     }
646 
647     /* Side 5 (Right) */
648     for (vz = 0; vz < faces[2]; vz++) {
649       for (vy = 0; vy < faces[1]; vy++) {
650         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0] + faces[0];
651         cone[0] = voffset + vertices[0] * vertices[1];
652         cone[1] = voffset;
653         cone[2] = voffset + vertices[0];
654         cone[3] = voffset + vertices[0] * vertices[1] + vertices[0];
655         PetscCall(DMPlexSetCone(dm, iface, cone));
656         PetscCall(DMSetLabelValue(dm, "marker", iface, markerRight));
657         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerRight));
658         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerRight));
659         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerRight));
660         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerRight));
661         iface++;
662       }
663     }
664   }
665   PetscCall(DMPlexSymmetrize(dm));
666   PetscCall(DMPlexStratify(dm));
667   /* Build coordinates */
668   PetscCall(DMSetCoordinateDim(dm, 3));
669   PetscCall(DMGetCoordinateSection(dm, &coordSection));
670   PetscCall(PetscSectionSetNumFields(coordSection, 1));
671   PetscCall(PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices));
672   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 3));
673   for (v = numFaces; v < numFaces + numVertices; ++v) {
674     PetscCall(PetscSectionSetDof(coordSection, v, 3));
675     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 3));
676   }
677   PetscCall(PetscSectionSetUp(coordSection));
678   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
679   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
680   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
681   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
682   PetscCall(VecSetBlockSize(coordinates, 3));
683   PetscCall(VecSetType(coordinates, VECSTANDARD));
684   PetscCall(VecGetArray(coordinates, &coords));
685   for (vz = 0; vz <= faces[2]; ++vz) {
686     for (vy = 0; vy <= faces[1]; ++vy) {
687       for (vx = 0; vx <= faces[0]; ++vx) {
688         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 0] = lower[0] + ((upper[0] - lower[0]) / faces[0]) * vx;
689         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 1] = lower[1] + ((upper[1] - lower[1]) / faces[1]) * vy;
690         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 2] = lower[2] + ((upper[2] - lower[2]) / faces[2]) * vz;
691       }
692     }
693   }
694   PetscCall(VecRestoreArray(coordinates, &coords));
695   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
696   PetscCall(VecDestroy(&coordinates));
697   PetscFunctionReturn(0);
698 }
699 
700 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate)
701 {
702   PetscFunctionBegin;
703   PetscValidLogicalCollectiveInt(dm, dim, 2);
704   PetscCall(DMSetDimension(dm, dim - 1));
705   PetscCall(DMSetCoordinateDim(dm, dim));
706   switch (dim) {
707   case 1:
708     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(dm, lower, upper, faces));
709     break;
710   case 2:
711     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(dm, lower, upper, faces));
712     break;
713   case 3:
714     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(dm, lower, upper, faces));
715     break;
716   default:
717     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Dimension not supported: %" PetscInt_FMT, dim);
718   }
719   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
720   PetscFunctionReturn(0);
721 }
722 
723 /*@C
724   DMPlexCreateBoxSurfaceMesh - Creates a mesh on the surface of the tensor product of unit intervals (box) using tensor cells (hexahedra).
725 
726   Collective
727 
728   Input Parameters:
729 + comm        - The communicator for the `DM` object
730 . dim         - The spatial dimension of the box, so the resulting mesh is has dimension dim-1
731 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
732 . lower       - The lower left corner, or NULL for (0, 0, 0)
733 . upper       - The upper right corner, or NULL for (1, 1, 1)
734 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
735 
736   Output Parameter:
737 . dm  - The `DM` object
738 
739   Level: beginner
740 
741 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateBoxMesh()`, `DMPlexCreateFromFile()`, `DMSetType()`, `DMCreate()`
742 @*/
743 PetscErrorCode DMPlexCreateBoxSurfaceMesh(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate, DM *dm)
744 {
745   PetscInt  fac[3] = {1, 1, 1};
746   PetscReal low[3] = {0, 0, 0};
747   PetscReal upp[3] = {1, 1, 1};
748 
749   PetscFunctionBegin;
750   PetscCall(DMCreate(comm, dm));
751   PetscCall(DMSetType(*dm, DMPLEX));
752   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(*dm, dim, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, interpolate));
753   PetscFunctionReturn(0);
754 }
755 
756 static PetscErrorCode DMPlexCreateLineMesh_Internal(DM dm, PetscInt segments, PetscReal lower, PetscReal upper, DMBoundaryType bd)
757 {
758   PetscInt     i, fStart, fEnd, numCells = 0, numVerts = 0;
759   PetscInt     numPoints[2], *coneSize, *cones, *coneOrientations;
760   PetscScalar *vertexCoords;
761   PetscReal    L, maxCell;
762   PetscBool    markerSeparate = PETSC_FALSE;
763   PetscInt     markerLeft = 1, faceMarkerLeft = 1;
764   PetscInt     markerRight = 1, faceMarkerRight = 2;
765   PetscBool    wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
766   PetscMPIInt  rank;
767 
768   PetscFunctionBegin;
769   PetscValidPointer(dm, 1);
770 
771   PetscCall(DMSetDimension(dm, 1));
772   PetscCall(DMCreateLabel(dm, "marker"));
773   PetscCall(DMCreateLabel(dm, "Face Sets"));
774 
775   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
776   if (rank == 0) numCells = segments;
777   if (rank == 0) numVerts = segments + (wrap ? 0 : 1);
778 
779   numPoints[0] = numVerts;
780   numPoints[1] = numCells;
781   PetscCall(PetscMalloc4(numCells + numVerts, &coneSize, numCells * 2, &cones, numCells + numVerts, &coneOrientations, numVerts, &vertexCoords));
782   PetscCall(PetscArrayzero(coneOrientations, numCells + numVerts));
783   for (i = 0; i < numCells; ++i) coneSize[i] = 2;
784   for (i = 0; i < numVerts; ++i) coneSize[numCells + i] = 0;
785   for (i = 0; i < numCells; ++i) {
786     cones[2 * i]     = numCells + i % numVerts;
787     cones[2 * i + 1] = numCells + (i + 1) % numVerts;
788   }
789   for (i = 0; i < numVerts; ++i) vertexCoords[i] = lower + (upper - lower) * ((PetscReal)i / (PetscReal)numCells);
790   PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
791   PetscCall(PetscFree4(coneSize, cones, coneOrientations, vertexCoords));
792 
793   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
794   if (markerSeparate) {
795     markerLeft  = faceMarkerLeft;
796     markerRight = faceMarkerRight;
797   }
798   if (!wrap && rank == 0) {
799     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
800     PetscCall(DMSetLabelValue(dm, "marker", fStart, markerLeft));
801     PetscCall(DMSetLabelValue(dm, "marker", fEnd - 1, markerRight));
802     PetscCall(DMSetLabelValue(dm, "Face Sets", fStart, faceMarkerLeft));
803     PetscCall(DMSetLabelValue(dm, "Face Sets", fEnd - 1, faceMarkerRight));
804   }
805   if (wrap) {
806     L       = upper - lower;
807     maxCell = (PetscReal)1.1 * (L / (PetscReal)PetscMax(1, segments));
808     PetscCall(DMSetPeriodicity(dm, &maxCell, &lower, &L));
809   }
810   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
811   PetscFunctionReturn(0);
812 }
813 
814 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
815 {
816   DM      boundary, vol;
817   DMLabel bdlabel;
818 
819   PetscFunctionBegin;
820   PetscValidPointer(dm, 1);
821   for (PetscInt i = 0; i < dim; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
822   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &boundary));
823   PetscCall(DMSetType(boundary, DMPLEX));
824   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(boundary, dim, faces, lower, upper, PETSC_FALSE));
825   PetscCall(DMPlexGenerate(boundary, NULL, interpolate, &vol));
826   PetscCall(DMGetLabel(vol, "marker", &bdlabel));
827   if (bdlabel) PetscCall(DMPlexLabelComplete(vol, bdlabel));
828   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_FALSE, vol));
829   PetscCall(DMPlexReplace_Internal(dm, &vol));
830   PetscCall(DMDestroy(&boundary));
831   PetscFunctionReturn(0);
832 }
833 
834 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
835 {
836   DMLabel     cutLabel  = NULL;
837   PetscInt    markerTop = 1, faceMarkerTop = 1;
838   PetscInt    markerBottom = 1, faceMarkerBottom = 1;
839   PetscInt    markerFront = 1, faceMarkerFront = 1;
840   PetscInt    markerBack = 1, faceMarkerBack = 1;
841   PetscInt    markerRight = 1, faceMarkerRight = 1;
842   PetscInt    markerLeft = 1, faceMarkerLeft = 1;
843   PetscInt    dim;
844   PetscBool   markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
845   PetscMPIInt rank;
846 
847   PetscFunctionBegin;
848   PetscCall(DMGetDimension(dm, &dim));
849   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
850   PetscCall(DMCreateLabel(dm, "marker"));
851   PetscCall(DMCreateLabel(dm, "Face Sets"));
852   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
853   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST || bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST || bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
854     if (cutMarker) {
855       PetscCall(DMCreateLabel(dm, "periodic_cut"));
856       PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
857     }
858   }
859   switch (dim) {
860   case 2:
861     faceMarkerTop    = 3;
862     faceMarkerBottom = 1;
863     faceMarkerRight  = 2;
864     faceMarkerLeft   = 4;
865     break;
866   case 3:
867     faceMarkerBottom = 1;
868     faceMarkerTop    = 2;
869     faceMarkerFront  = 3;
870     faceMarkerBack   = 4;
871     faceMarkerRight  = 5;
872     faceMarkerLeft   = 6;
873     break;
874   default:
875     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
876   }
877   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
878   if (markerSeparate) {
879     markerBottom = faceMarkerBottom;
880     markerTop    = faceMarkerTop;
881     markerFront  = faceMarkerFront;
882     markerBack   = faceMarkerBack;
883     markerRight  = faceMarkerRight;
884     markerLeft   = faceMarkerLeft;
885   }
886   {
887     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
888     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
889     const PetscInt numZEdges    = rank == 0 ? edges[2] : 0;
890     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0] + 1) : 0;
891     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1] + 1) : 0;
892     const PetscInt numZVertices = rank == 0 ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2] + 1) : 0;
893     const PetscInt numCells     = numXEdges * numYEdges * numZEdges;
894     const PetscInt numXFaces    = numYEdges * numZEdges;
895     const PetscInt numYFaces    = numXEdges * numZEdges;
896     const PetscInt numZFaces    = numXEdges * numYEdges;
897     const PetscInt numTotXFaces = numXVertices * numXFaces;
898     const PetscInt numTotYFaces = numYVertices * numYFaces;
899     const PetscInt numTotZFaces = numZVertices * numZFaces;
900     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
901     const PetscInt numTotXEdges = numXEdges * numYVertices * numZVertices;
902     const PetscInt numTotYEdges = numYEdges * numXVertices * numZVertices;
903     const PetscInt numTotZEdges = numZEdges * numXVertices * numYVertices;
904     const PetscInt numVertices  = numXVertices * numYVertices * numZVertices;
905     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
906     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
907     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
908     const PetscInt firstYFace   = firstXFace + numTotXFaces;
909     const PetscInt firstZFace   = firstYFace + numTotYFaces;
910     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
911     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
912     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
913     Vec            coordinates;
914     PetscSection   coordSection;
915     PetscScalar   *coords;
916     PetscInt       coordSize;
917     PetscInt       v, vx, vy, vz;
918     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
919 
920     PetscCall(DMPlexSetChart(dm, 0, numCells + numFaces + numEdges + numVertices));
921     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
922     for (f = firstXFace; f < firstXFace + numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
923     for (e = firstXEdge; e < firstXEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
924     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
925     /* Build cells */
926     for (fz = 0; fz < numZEdges; ++fz) {
927       for (fy = 0; fy < numYEdges; ++fy) {
928         for (fx = 0; fx < numXEdges; ++fx) {
929           PetscInt cell  = (fz * numYEdges + fy) * numXEdges + fx;
930           PetscInt faceB = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
931           PetscInt faceT = firstZFace + (fy * numXEdges + fx) * numZVertices + ((fz + 1) % numZVertices);
932           PetscInt faceF = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
933           PetscInt faceK = firstYFace + (fz * numXEdges + fx) * numYVertices + ((fy + 1) % numYVertices);
934           PetscInt faceL = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
935           PetscInt faceR = firstXFace + (fz * numYEdges + fy) * numXVertices + ((fx + 1) % numXVertices);
936           /* B,  T,  F,  K,  R,  L */
937           PetscInt ornt[6] = {-2, 0, 0, -3, 0, -2}; /* ??? */
938           PetscInt cone[6];
939 
940           /* no boundary twisting in 3D */
941           cone[0] = faceB;
942           cone[1] = faceT;
943           cone[2] = faceF;
944           cone[3] = faceK;
945           cone[4] = faceR;
946           cone[5] = faceL;
947           PetscCall(DMPlexSetCone(dm, cell, cone));
948           PetscCall(DMPlexSetConeOrientation(dm, cell, ornt));
949           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
950           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
951           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
952         }
953       }
954     }
955     /* Build x faces */
956     for (fz = 0; fz < numZEdges; ++fz) {
957       for (fy = 0; fy < numYEdges; ++fy) {
958         for (fx = 0; fx < numXVertices; ++fx) {
959           PetscInt face    = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
960           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
961           PetscInt edgeR   = firstZEdge + (((fy + 1) % numYVertices) * numXVertices + fx) * numZEdges + fz;
962           PetscInt edgeB   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
963           PetscInt edgeT   = firstYEdge + (((fz + 1) % numZVertices) * numXVertices + fx) * numYEdges + fy;
964           PetscInt ornt[4] = {0, 0, -1, -1};
965           PetscInt cone[4];
966 
967           if (dim == 3) {
968             /* markers */
969             if (bdX != DM_BOUNDARY_PERIODIC) {
970               if (fx == numXVertices - 1) {
971                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight));
972                 PetscCall(DMSetLabelValue(dm, "marker", face, markerRight));
973               } else if (fx == 0) {
974                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft));
975                 PetscCall(DMSetLabelValue(dm, "marker", face, markerLeft));
976               }
977             }
978           }
979           cone[0] = edgeB;
980           cone[1] = edgeR;
981           cone[2] = edgeT;
982           cone[3] = edgeL;
983           PetscCall(DMPlexSetCone(dm, face, cone));
984           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
985         }
986       }
987     }
988     /* Build y faces */
989     for (fz = 0; fz < numZEdges; ++fz) {
990       for (fx = 0; fx < numXEdges; ++fx) {
991         for (fy = 0; fy < numYVertices; ++fy) {
992           PetscInt face    = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
993           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
994           PetscInt edgeR   = firstZEdge + (fy * numXVertices + ((fx + 1) % numXVertices)) * numZEdges + fz;
995           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
996           PetscInt edgeT   = firstXEdge + (((fz + 1) % numZVertices) * numYVertices + fy) * numXEdges + fx;
997           PetscInt ornt[4] = {0, 0, -1, -1};
998           PetscInt cone[4];
999 
1000           if (dim == 3) {
1001             /* markers */
1002             if (bdY != DM_BOUNDARY_PERIODIC) {
1003               if (fy == numYVertices - 1) {
1004                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack));
1005                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBack));
1006               } else if (fy == 0) {
1007                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront));
1008                 PetscCall(DMSetLabelValue(dm, "marker", face, markerFront));
1009               }
1010             }
1011           }
1012           cone[0] = edgeB;
1013           cone[1] = edgeR;
1014           cone[2] = edgeT;
1015           cone[3] = edgeL;
1016           PetscCall(DMPlexSetCone(dm, face, cone));
1017           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1018         }
1019       }
1020     }
1021     /* Build z faces */
1022     for (fy = 0; fy < numYEdges; ++fy) {
1023       for (fx = 0; fx < numXEdges; ++fx) {
1024         for (fz = 0; fz < numZVertices; fz++) {
1025           PetscInt face    = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
1026           PetscInt edgeL   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
1027           PetscInt edgeR   = firstYEdge + (fz * numXVertices + ((fx + 1) % numXVertices)) * numYEdges + fy;
1028           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1029           PetscInt edgeT   = firstXEdge + (fz * numYVertices + ((fy + 1) % numYVertices)) * numXEdges + fx;
1030           PetscInt ornt[4] = {0, 0, -1, -1};
1031           PetscInt cone[4];
1032 
1033           if (dim == 2) {
1034             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges - 1) {
1035               edgeR += numYEdges - 1 - 2 * fy;
1036               ornt[1] = -1;
1037             }
1038             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges - 1) {
1039               edgeT += numXEdges - 1 - 2 * fx;
1040               ornt[2] = 0;
1041             }
1042             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1043             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1044           } else {
1045             /* markers */
1046             if (bdZ != DM_BOUNDARY_PERIODIC) {
1047               if (fz == numZVertices - 1) {
1048                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop));
1049                 PetscCall(DMSetLabelValue(dm, "marker", face, markerTop));
1050               } else if (fz == 0) {
1051                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom));
1052                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBottom));
1053               }
1054             }
1055           }
1056           cone[0] = edgeB;
1057           cone[1] = edgeR;
1058           cone[2] = edgeT;
1059           cone[3] = edgeL;
1060           PetscCall(DMPlexSetCone(dm, face, cone));
1061           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1062         }
1063       }
1064     }
1065     /* Build Z edges*/
1066     for (vy = 0; vy < numYVertices; vy++) {
1067       for (vx = 0; vx < numXVertices; vx++) {
1068         for (ez = 0; ez < numZEdges; ez++) {
1069           const PetscInt edge    = firstZEdge + (vy * numXVertices + vx) * numZEdges + ez;
1070           const PetscInt vertexB = firstVertex + (ez * numYVertices + vy) * numXVertices + vx;
1071           const PetscInt vertexT = firstVertex + (((ez + 1) % numZVertices) * numYVertices + vy) * numXVertices + vx;
1072           PetscInt       cone[2];
1073 
1074           cone[0] = vertexB;
1075           cone[1] = vertexT;
1076           PetscCall(DMPlexSetCone(dm, edge, cone));
1077           if (dim == 3) {
1078             if (bdX != DM_BOUNDARY_PERIODIC) {
1079               if (vx == numXVertices - 1) {
1080                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1081                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1082                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1083               } else if (vx == 0) {
1084                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1085                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1086                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1087               }
1088             }
1089             if (bdY != DM_BOUNDARY_PERIODIC) {
1090               if (vy == numYVertices - 1) {
1091                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1092                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1093                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1094               } else if (vy == 0) {
1095                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1096                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1097                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1098               }
1099             }
1100           }
1101         }
1102       }
1103     }
1104     /* Build Y edges*/
1105     for (vz = 0; vz < numZVertices; vz++) {
1106       for (vx = 0; vx < numXVertices; vx++) {
1107         for (ey = 0; ey < numYEdges; ey++) {
1108           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges - 1) ? (numXVertices - vx - 1) : (vz * numYVertices + ((ey + 1) % numYVertices)) * numXVertices + vx;
1109           const PetscInt edge    = firstYEdge + (vz * numXVertices + vx) * numYEdges + ey;
1110           const PetscInt vertexF = firstVertex + (vz * numYVertices + ey) * numXVertices + vx;
1111           const PetscInt vertexK = firstVertex + nextv;
1112           PetscInt       cone[2];
1113 
1114           cone[0] = vertexF;
1115           cone[1] = vertexK;
1116           PetscCall(DMPlexSetCone(dm, edge, cone));
1117           if (dim == 2) {
1118             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1119               if (vx == numXVertices - 1) {
1120                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight));
1121                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1122                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1123                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1124               } else if (vx == 0) {
1125                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft));
1126                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1127                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1128                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1129               }
1130             } else {
1131               if (vx == 0 && cutLabel) {
1132                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1133                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1134                 if (ey == numYEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1135               }
1136             }
1137           } else {
1138             if (bdX != DM_BOUNDARY_PERIODIC) {
1139               if (vx == numXVertices - 1) {
1140                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1141                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1142                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1143               } else if (vx == 0) {
1144                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1145                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1146                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1147               }
1148             }
1149             if (bdZ != DM_BOUNDARY_PERIODIC) {
1150               if (vz == numZVertices - 1) {
1151                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1152                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1153                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1154               } else if (vz == 0) {
1155                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1156                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1157                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1158               }
1159             }
1160           }
1161         }
1162       }
1163     }
1164     /* Build X edges*/
1165     for (vz = 0; vz < numZVertices; vz++) {
1166       for (vy = 0; vy < numYVertices; vy++) {
1167         for (ex = 0; ex < numXEdges; ex++) {
1168           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges - 1) ? (numYVertices - vy - 1) * numXVertices : (vz * numYVertices + vy) * numXVertices + (ex + 1) % numXVertices;
1169           const PetscInt edge    = firstXEdge + (vz * numYVertices + vy) * numXEdges + ex;
1170           const PetscInt vertexL = firstVertex + (vz * numYVertices + vy) * numXVertices + ex;
1171           const PetscInt vertexR = firstVertex + nextv;
1172           PetscInt       cone[2];
1173 
1174           cone[0] = vertexL;
1175           cone[1] = vertexR;
1176           PetscCall(DMPlexSetCone(dm, edge, cone));
1177           if (dim == 2) {
1178             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1179               if (vy == numYVertices - 1) {
1180                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop));
1181                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1182                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1183                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1184               } else if (vy == 0) {
1185                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom));
1186                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1187                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1188                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1189               }
1190             } else {
1191               if (vy == 0 && cutLabel) {
1192                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1193                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1194                 if (ex == numXEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1195               }
1196             }
1197           } else {
1198             if (bdY != DM_BOUNDARY_PERIODIC) {
1199               if (vy == numYVertices - 1) {
1200                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1201                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1202                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1203               } else if (vy == 0) {
1204                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1205                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1206                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1207               }
1208             }
1209             if (bdZ != DM_BOUNDARY_PERIODIC) {
1210               if (vz == numZVertices - 1) {
1211                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1212                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1213                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1214               } else if (vz == 0) {
1215                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1216                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1217                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1218               }
1219             }
1220           }
1221         }
1222       }
1223     }
1224     PetscCall(DMPlexSymmetrize(dm));
1225     PetscCall(DMPlexStratify(dm));
1226     /* Build coordinates */
1227     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1228     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1229     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1230     PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVertices));
1231     for (v = firstVertex; v < firstVertex + numVertices; ++v) {
1232       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1233       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1234     }
1235     PetscCall(PetscSectionSetUp(coordSection));
1236     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1237     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1238     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1239     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1240     PetscCall(VecSetBlockSize(coordinates, dim));
1241     PetscCall(VecSetType(coordinates, VECSTANDARD));
1242     PetscCall(VecGetArray(coordinates, &coords));
1243     for (vz = 0; vz < numZVertices; ++vz) {
1244       for (vy = 0; vy < numYVertices; ++vy) {
1245         for (vx = 0; vx < numXVertices; ++vx) {
1246           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * vx;
1247           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * vy;
1248           if (dim == 3) coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 2] = lower[2] + ((upper[2] - lower[2]) / numZEdges) * vz;
1249         }
1250       }
1251     }
1252     PetscCall(VecRestoreArray(coordinates, &coords));
1253     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1254     PetscCall(VecDestroy(&coordinates));
1255   }
1256   PetscFunctionReturn(0);
1257 }
1258 
1259 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1260 {
1261   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1262   PetscInt       fac[3] = {0, 0, 0}, d;
1263 
1264   PetscFunctionBegin;
1265   PetscValidPointer(dm, 1);
1266   PetscValidLogicalCollectiveInt(dm, dim, 2);
1267   PetscCall(DMSetDimension(dm, dim));
1268   for (d = 0; d < dim; ++d) {
1269     fac[d] = faces[d];
1270     bdt[d] = periodicity[d];
1271   }
1272   PetscCall(DMPlexCreateCubeMesh_Internal(dm, lower, upper, fac, bdt[0], bdt[1], bdt[2]));
1273   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST || periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST || (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
1274     PetscReal L[3]       = {-1., -1., 0.};
1275     PetscReal maxCell[3] = {-1., -1., 0.};
1276 
1277     for (d = 0; d < dim; ++d) {
1278       if (periodicity[d] != DM_BOUNDARY_NONE) {
1279         L[d]       = upper[d] - lower[d];
1280         maxCell[d] = 1.1 * (L[d] / PetscMax(1, faces[d]));
1281       }
1282     }
1283     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1284   }
1285   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
1286   PetscFunctionReturn(0);
1287 }
1288 
1289 static PetscErrorCode DMPlexCreateBoxMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1290 {
1291   PetscFunctionBegin;
1292   if (dim == 1) PetscCall(DMPlexCreateLineMesh_Internal(dm, faces[0], lower[0], upper[0], periodicity[0]));
1293   else if (simplex) PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1294   else PetscCall(DMPlexCreateBoxMesh_Tensor_Internal(dm, dim, faces, lower, upper, periodicity));
1295   if (!interpolate && dim > 1 && !simplex) {
1296     DM udm;
1297 
1298     PetscCall(DMPlexUninterpolate(dm, &udm));
1299     PetscCall(DMPlexCopyCoordinates(dm, udm));
1300     PetscCall(DMPlexReplace_Internal(dm, &udm));
1301   }
1302   PetscFunctionReturn(0);
1303 }
1304 
1305 /*@C
1306   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
1307 
1308   Collective
1309 
1310   Input Parameters:
1311 + comm        - The communicator for the `DM` object
1312 . dim         - The spatial dimension
1313 . simplex     - `PETSC_TRUE` for simplices, `PETSC_FALSE` for tensor cells
1314 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1315 . lower       - The lower left corner, or NULL for (0, 0, 0)
1316 . upper       - The upper right corner, or NULL for (1, 1, 1)
1317 . periodicity - The boundary type for the X,Y,Z direction, or NULL for `DM_BOUNDARY_NONE`
1318 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1319 
1320   Output Parameter:
1321 . dm  - The `DM` object
1322 
1323   Level: beginner
1324 
1325   Note:
1326    To customize this mesh using options, use
1327 .vb
1328   DMCreate(comm, &dm);
1329   DMSetType(dm, DMPLEX);
1330   DMSetFromOptions(dm);
1331 .ve
1332 and use the options in `DMSetFromOptions()`.
1333 
1334   Here is the numbering returned for 2 faces in each direction for tensor cells:
1335 .vb
1336  10---17---11---18----12
1337   |         |         |
1338   |         |         |
1339  20    2   22    3    24
1340   |         |         |
1341   |         |         |
1342   7---15----8---16----9
1343   |         |         |
1344   |         |         |
1345  19    0   21    1   23
1346   |         |         |
1347   |         |         |
1348   4---13----5---14----6
1349 .ve
1350 and for simplicial cells
1351 .vb
1352  14----8---15----9----16
1353   |\     5  |\      7 |
1354   | \       | \       |
1355  13   2    14    3    15
1356   | 4   \   | 6   \   |
1357   |       \ |       \ |
1358  11----6---12----7----13
1359   |\        |\        |
1360   | \    1  | \     3 |
1361  10   0    11    1    12
1362   | 0   \   | 2   \   |
1363   |       \ |       \ |
1364   8----4----9----5----10
1365 .ve
1366 
1367 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
1368 @*/
1369 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
1370 {
1371   PetscInt       fac[3] = {1, 1, 1};
1372   PetscReal      low[3] = {0, 0, 0};
1373   PetscReal      upp[3] = {1, 1, 1};
1374   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1375 
1376   PetscFunctionBegin;
1377   PetscCall(DMCreate(comm, dm));
1378   PetscCall(DMSetType(*dm, DMPLEX));
1379   PetscCall(DMPlexCreateBoxMesh_Internal(*dm, dim, simplex, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt, interpolate));
1380   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1381   PetscFunctionReturn(0);
1382 }
1383 
1384 static PetscErrorCode DMPlexCreateWedgeBoxMesh_Internal(DM dm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1385 {
1386   DM       bdm, vol;
1387   PetscInt i;
1388 
1389   PetscFunctionBegin;
1390   for (i = 0; i < 3; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity not yet supported");
1391   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &bdm));
1392   PetscCall(DMSetType(bdm, DMPLEX));
1393   PetscCall(DMSetDimension(bdm, 2));
1394   PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(bdm, 2, faces, lower, upper, periodicity, PETSC_TRUE));
1395   PetscCall(DMPlexExtrude(bdm, faces[2], upper[2] - lower[2], PETSC_TRUE, PETSC_FALSE, NULL, NULL, &vol));
1396   PetscCall(DMDestroy(&bdm));
1397   PetscCall(DMPlexReplace_Internal(dm, &vol));
1398   if (lower[2] != 0.0) {
1399     Vec          v;
1400     PetscScalar *x;
1401     PetscInt     cDim, n;
1402 
1403     PetscCall(DMGetCoordinatesLocal(dm, &v));
1404     PetscCall(VecGetBlockSize(v, &cDim));
1405     PetscCall(VecGetLocalSize(v, &n));
1406     PetscCall(VecGetArray(v, &x));
1407     x += cDim;
1408     for (i = 0; i < n; i += cDim) x[i] += lower[2];
1409     PetscCall(VecRestoreArray(v, &x));
1410     PetscCall(DMSetCoordinatesLocal(dm, v));
1411   }
1412   PetscFunctionReturn(0);
1413 }
1414 
1415 /*@
1416   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tesselating the (x,y) plane and extruding in the third direction using wedge cells.
1417 
1418   Collective
1419 
1420   Input Parameters:
1421 + comm        - The communicator for the `DM` object
1422 . faces       - Number of faces per dimension, or NULL for (1, 1, 1)
1423 . lower       - The lower left corner, or NULL for (0, 0, 0)
1424 . upper       - The upper right corner, or NULL for (1, 1, 1)
1425 . periodicity - The boundary type for the X,Y,Z direction, or NULL for `DM_BOUNDARY_NONE`
1426 . orderHeight - If `PETSC_TRUE`, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1427 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1428 
1429   Output Parameter:
1430 . dm  - The `DM` object
1431 
1432   Level: beginner
1433 
1434 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateWedgeCylinderMesh()`, `DMExtrude()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1435 @*/
1436 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool orderHeight, PetscBool interpolate, DM *dm)
1437 {
1438   PetscInt       fac[3] = {1, 1, 1};
1439   PetscReal      low[3] = {0, 0, 0};
1440   PetscReal      upp[3] = {1, 1, 1};
1441   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1442 
1443   PetscFunctionBegin;
1444   PetscCall(DMCreate(comm, dm));
1445   PetscCall(DMSetType(*dm, DMPLEX));
1446   PetscCall(DMPlexCreateWedgeBoxMesh_Internal(*dm, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt));
1447   if (!interpolate) {
1448     DM udm;
1449 
1450     PetscCall(DMPlexUninterpolate(*dm, &udm));
1451     PetscCall(DMPlexReplace_Internal(*dm, &udm));
1452   }
1453   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1454   PetscFunctionReturn(0);
1455 }
1456 
1457 /*@C
1458   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all `DM` options in the database.
1459 
1460   Logically Collective on dm
1461 
1462   Input Parameters:
1463 + dm - the DM context
1464 - prefix - the prefix to prepend to all option names
1465 
1466   Level: advanced
1467 
1468   Note:
1469   A hyphen (-) must NOT be given at the beginning of the prefix name.
1470   The first character of all runtime options is AUTOMATICALLY the hyphen.
1471 
1472 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `SNESSetFromOptions()`
1473 @*/
1474 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
1475 {
1476   DM_Plex *mesh = (DM_Plex *)dm->data;
1477 
1478   PetscFunctionBegin;
1479   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1480   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, prefix));
1481   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)mesh->partitioner, prefix));
1482   PetscFunctionReturn(0);
1483 }
1484 
1485 /* Remap geometry to cylinder
1486    TODO: This only works for a single refinement, then it is broken
1487 
1488      Interior square: Linear interpolation is correct
1489      The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
1490      such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
1491 
1492        phi     = arctan(y/x)
1493        d_close = sqrt(1/8 + 1/4 sin^2(phi))
1494        d_far   = sqrt(1/2 + sin^2(phi))
1495 
1496      so we remap them using
1497 
1498        x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
1499        y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
1500 
1501      If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
1502 */
1503 static void snapToCylinder(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
1504 {
1505   const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1506   const PetscReal ds2 = 0.5 * dis;
1507 
1508   if ((PetscAbsScalar(u[0]) <= ds2) && (PetscAbsScalar(u[1]) <= ds2)) {
1509     f0[0] = u[0];
1510     f0[1] = u[1];
1511   } else {
1512     PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
1513 
1514     x    = PetscRealPart(u[0]);
1515     y    = PetscRealPart(u[1]);
1516     phi  = PetscAtan2Real(y, x);
1517     sinp = PetscSinReal(phi);
1518     cosp = PetscCosReal(phi);
1519     if ((PetscAbsReal(phi) > PETSC_PI / 4.0) && (PetscAbsReal(phi) < 3.0 * PETSC_PI / 4.0)) {
1520       dc = PetscAbsReal(ds2 / sinp);
1521       df = PetscAbsReal(dis / sinp);
1522       xc = ds2 * x / PetscAbsReal(y);
1523       yc = ds2 * PetscSignReal(y);
1524     } else {
1525       dc = PetscAbsReal(ds2 / cosp);
1526       df = PetscAbsReal(dis / cosp);
1527       xc = ds2 * PetscSignReal(x);
1528       yc = ds2 * y / PetscAbsReal(x);
1529     }
1530     f0[0] = xc + (u[0] - xc) * (1.0 - dc) / (df - dc);
1531     f0[1] = yc + (u[1] - yc) * (1.0 - dc) / (df - dc);
1532   }
1533   f0[2] = u[2];
1534 }
1535 
1536 static PetscErrorCode DMPlexCreateHexCylinderMesh_Internal(DM dm, DMBoundaryType periodicZ)
1537 {
1538   const PetscInt dim = 3;
1539   PetscInt       numCells, numVertices;
1540   PetscMPIInt    rank;
1541 
1542   PetscFunctionBegin;
1543   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1544   PetscCall(DMSetDimension(dm, dim));
1545   /* Create topology */
1546   {
1547     PetscInt cone[8], c;
1548 
1549     numCells    = rank == 0 ? 5 : 0;
1550     numVertices = rank == 0 ? 16 : 0;
1551     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1552       numCells *= 3;
1553       numVertices = rank == 0 ? 24 : 0;
1554     }
1555     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1556     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 8));
1557     PetscCall(DMSetUp(dm));
1558     if (rank == 0) {
1559       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1560         cone[0] = 15;
1561         cone[1] = 18;
1562         cone[2] = 17;
1563         cone[3] = 16;
1564         cone[4] = 31;
1565         cone[5] = 32;
1566         cone[6] = 33;
1567         cone[7] = 34;
1568         PetscCall(DMPlexSetCone(dm, 0, cone));
1569         cone[0] = 16;
1570         cone[1] = 17;
1571         cone[2] = 24;
1572         cone[3] = 23;
1573         cone[4] = 32;
1574         cone[5] = 36;
1575         cone[6] = 37;
1576         cone[7] = 33; /* 22 25 26 21 */
1577         PetscCall(DMPlexSetCone(dm, 1, cone));
1578         cone[0] = 18;
1579         cone[1] = 27;
1580         cone[2] = 24;
1581         cone[3] = 17;
1582         cone[4] = 34;
1583         cone[5] = 33;
1584         cone[6] = 37;
1585         cone[7] = 38;
1586         PetscCall(DMPlexSetCone(dm, 2, cone));
1587         cone[0] = 29;
1588         cone[1] = 27;
1589         cone[2] = 18;
1590         cone[3] = 15;
1591         cone[4] = 35;
1592         cone[5] = 31;
1593         cone[6] = 34;
1594         cone[7] = 38;
1595         PetscCall(DMPlexSetCone(dm, 3, cone));
1596         cone[0] = 29;
1597         cone[1] = 15;
1598         cone[2] = 16;
1599         cone[3] = 23;
1600         cone[4] = 35;
1601         cone[5] = 36;
1602         cone[6] = 32;
1603         cone[7] = 31;
1604         PetscCall(DMPlexSetCone(dm, 4, cone));
1605 
1606         cone[0] = 31;
1607         cone[1] = 34;
1608         cone[2] = 33;
1609         cone[3] = 32;
1610         cone[4] = 19;
1611         cone[5] = 22;
1612         cone[6] = 21;
1613         cone[7] = 20;
1614         PetscCall(DMPlexSetCone(dm, 5, cone));
1615         cone[0] = 32;
1616         cone[1] = 33;
1617         cone[2] = 37;
1618         cone[3] = 36;
1619         cone[4] = 22;
1620         cone[5] = 25;
1621         cone[6] = 26;
1622         cone[7] = 21;
1623         PetscCall(DMPlexSetCone(dm, 6, cone));
1624         cone[0] = 34;
1625         cone[1] = 38;
1626         cone[2] = 37;
1627         cone[3] = 33;
1628         cone[4] = 20;
1629         cone[5] = 21;
1630         cone[6] = 26;
1631         cone[7] = 28;
1632         PetscCall(DMPlexSetCone(dm, 7, cone));
1633         cone[0] = 35;
1634         cone[1] = 38;
1635         cone[2] = 34;
1636         cone[3] = 31;
1637         cone[4] = 30;
1638         cone[5] = 19;
1639         cone[6] = 20;
1640         cone[7] = 28;
1641         PetscCall(DMPlexSetCone(dm, 8, cone));
1642         cone[0] = 35;
1643         cone[1] = 31;
1644         cone[2] = 32;
1645         cone[3] = 36;
1646         cone[4] = 30;
1647         cone[5] = 25;
1648         cone[6] = 22;
1649         cone[7] = 19;
1650         PetscCall(DMPlexSetCone(dm, 9, cone));
1651 
1652         cone[0] = 19;
1653         cone[1] = 20;
1654         cone[2] = 21;
1655         cone[3] = 22;
1656         cone[4] = 15;
1657         cone[5] = 16;
1658         cone[6] = 17;
1659         cone[7] = 18;
1660         PetscCall(DMPlexSetCone(dm, 10, cone));
1661         cone[0] = 22;
1662         cone[1] = 21;
1663         cone[2] = 26;
1664         cone[3] = 25;
1665         cone[4] = 16;
1666         cone[5] = 23;
1667         cone[6] = 24;
1668         cone[7] = 17;
1669         PetscCall(DMPlexSetCone(dm, 11, cone));
1670         cone[0] = 20;
1671         cone[1] = 28;
1672         cone[2] = 26;
1673         cone[3] = 21;
1674         cone[4] = 18;
1675         cone[5] = 17;
1676         cone[6] = 24;
1677         cone[7] = 27;
1678         PetscCall(DMPlexSetCone(dm, 12, cone));
1679         cone[0] = 30;
1680         cone[1] = 28;
1681         cone[2] = 20;
1682         cone[3] = 19;
1683         cone[4] = 29;
1684         cone[5] = 15;
1685         cone[6] = 18;
1686         cone[7] = 27;
1687         PetscCall(DMPlexSetCone(dm, 13, cone));
1688         cone[0] = 30;
1689         cone[1] = 19;
1690         cone[2] = 22;
1691         cone[3] = 25;
1692         cone[4] = 29;
1693         cone[5] = 23;
1694         cone[6] = 16;
1695         cone[7] = 15;
1696         PetscCall(DMPlexSetCone(dm, 14, cone));
1697       } else {
1698         cone[0] = 5;
1699         cone[1] = 8;
1700         cone[2] = 7;
1701         cone[3] = 6;
1702         cone[4] = 9;
1703         cone[5] = 12;
1704         cone[6] = 11;
1705         cone[7] = 10;
1706         PetscCall(DMPlexSetCone(dm, 0, cone));
1707         cone[0] = 6;
1708         cone[1] = 7;
1709         cone[2] = 14;
1710         cone[3] = 13;
1711         cone[4] = 12;
1712         cone[5] = 15;
1713         cone[6] = 16;
1714         cone[7] = 11;
1715         PetscCall(DMPlexSetCone(dm, 1, cone));
1716         cone[0] = 8;
1717         cone[1] = 17;
1718         cone[2] = 14;
1719         cone[3] = 7;
1720         cone[4] = 10;
1721         cone[5] = 11;
1722         cone[6] = 16;
1723         cone[7] = 18;
1724         PetscCall(DMPlexSetCone(dm, 2, cone));
1725         cone[0] = 19;
1726         cone[1] = 17;
1727         cone[2] = 8;
1728         cone[3] = 5;
1729         cone[4] = 20;
1730         cone[5] = 9;
1731         cone[6] = 10;
1732         cone[7] = 18;
1733         PetscCall(DMPlexSetCone(dm, 3, cone));
1734         cone[0] = 19;
1735         cone[1] = 5;
1736         cone[2] = 6;
1737         cone[3] = 13;
1738         cone[4] = 20;
1739         cone[5] = 15;
1740         cone[6] = 12;
1741         cone[7] = 9;
1742         PetscCall(DMPlexSetCone(dm, 4, cone));
1743       }
1744     }
1745     PetscCall(DMPlexSymmetrize(dm));
1746     PetscCall(DMPlexStratify(dm));
1747   }
1748   /* Create cube geometry */
1749   {
1750     Vec             coordinates;
1751     PetscSection    coordSection;
1752     PetscScalar    *coords;
1753     PetscInt        coordSize, v;
1754     const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1755     const PetscReal ds2 = dis / 2.0;
1756 
1757     /* Build coordinates */
1758     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1759     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1760     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1761     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1762     for (v = numCells; v < numCells + numVertices; ++v) {
1763       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1764       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1765     }
1766     PetscCall(PetscSectionSetUp(coordSection));
1767     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1768     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1769     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1770     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1771     PetscCall(VecSetBlockSize(coordinates, dim));
1772     PetscCall(VecSetType(coordinates, VECSTANDARD));
1773     PetscCall(VecGetArray(coordinates, &coords));
1774     if (rank == 0) {
1775       coords[0 * dim + 0]  = -ds2;
1776       coords[0 * dim + 1]  = -ds2;
1777       coords[0 * dim + 2]  = 0.0;
1778       coords[1 * dim + 0]  = ds2;
1779       coords[1 * dim + 1]  = -ds2;
1780       coords[1 * dim + 2]  = 0.0;
1781       coords[2 * dim + 0]  = ds2;
1782       coords[2 * dim + 1]  = ds2;
1783       coords[2 * dim + 2]  = 0.0;
1784       coords[3 * dim + 0]  = -ds2;
1785       coords[3 * dim + 1]  = ds2;
1786       coords[3 * dim + 2]  = 0.0;
1787       coords[4 * dim + 0]  = -ds2;
1788       coords[4 * dim + 1]  = -ds2;
1789       coords[4 * dim + 2]  = 1.0;
1790       coords[5 * dim + 0]  = -ds2;
1791       coords[5 * dim + 1]  = ds2;
1792       coords[5 * dim + 2]  = 1.0;
1793       coords[6 * dim + 0]  = ds2;
1794       coords[6 * dim + 1]  = ds2;
1795       coords[6 * dim + 2]  = 1.0;
1796       coords[7 * dim + 0]  = ds2;
1797       coords[7 * dim + 1]  = -ds2;
1798       coords[7 * dim + 2]  = 1.0;
1799       coords[8 * dim + 0]  = dis;
1800       coords[8 * dim + 1]  = -dis;
1801       coords[8 * dim + 2]  = 0.0;
1802       coords[9 * dim + 0]  = dis;
1803       coords[9 * dim + 1]  = dis;
1804       coords[9 * dim + 2]  = 0.0;
1805       coords[10 * dim + 0] = dis;
1806       coords[10 * dim + 1] = -dis;
1807       coords[10 * dim + 2] = 1.0;
1808       coords[11 * dim + 0] = dis;
1809       coords[11 * dim + 1] = dis;
1810       coords[11 * dim + 2] = 1.0;
1811       coords[12 * dim + 0] = -dis;
1812       coords[12 * dim + 1] = dis;
1813       coords[12 * dim + 2] = 0.0;
1814       coords[13 * dim + 0] = -dis;
1815       coords[13 * dim + 1] = dis;
1816       coords[13 * dim + 2] = 1.0;
1817       coords[14 * dim + 0] = -dis;
1818       coords[14 * dim + 1] = -dis;
1819       coords[14 * dim + 2] = 0.0;
1820       coords[15 * dim + 0] = -dis;
1821       coords[15 * dim + 1] = -dis;
1822       coords[15 * dim + 2] = 1.0;
1823       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1824         /* 15 31 19 */ coords[16 * dim + 0] = -ds2;
1825         coords[16 * dim + 1]                = -ds2;
1826         coords[16 * dim + 2]                = 0.5;
1827         /* 16 32 22 */ coords[17 * dim + 0] = ds2;
1828         coords[17 * dim + 1]                = -ds2;
1829         coords[17 * dim + 2]                = 0.5;
1830         /* 17 33 21 */ coords[18 * dim + 0] = ds2;
1831         coords[18 * dim + 1]                = ds2;
1832         coords[18 * dim + 2]                = 0.5;
1833         /* 18 34 20 */ coords[19 * dim + 0] = -ds2;
1834         coords[19 * dim + 1]                = ds2;
1835         coords[19 * dim + 2]                = 0.5;
1836         /* 29 35 30 */ coords[20 * dim + 0] = -dis;
1837         coords[20 * dim + 1]                = -dis;
1838         coords[20 * dim + 2]                = 0.5;
1839         /* 23 36 25 */ coords[21 * dim + 0] = dis;
1840         coords[21 * dim + 1]                = -dis;
1841         coords[21 * dim + 2]                = 0.5;
1842         /* 24 37 26 */ coords[22 * dim + 0] = dis;
1843         coords[22 * dim + 1]                = dis;
1844         coords[22 * dim + 2]                = 0.5;
1845         /* 27 38 28 */ coords[23 * dim + 0] = -dis;
1846         coords[23 * dim + 1]                = dis;
1847         coords[23 * dim + 2]                = 0.5;
1848       }
1849     }
1850     PetscCall(VecRestoreArray(coordinates, &coords));
1851     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1852     PetscCall(VecDestroy(&coordinates));
1853   }
1854   /* Create periodicity */
1855   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1856     PetscReal L[3]       = {-1., -1., 0.};
1857     PetscReal maxCell[3] = {-1., -1., 0.};
1858     PetscReal lower[3]   = {0.0, 0.0, 0.0};
1859     PetscReal upper[3]   = {1.0, 1.0, 1.5};
1860     PetscInt  numZCells  = 3;
1861 
1862     L[2]       = upper[2] - lower[2];
1863     maxCell[2] = 1.1 * (L[2] / numZCells);
1864     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1865   }
1866   {
1867     DM          cdm;
1868     PetscDS     cds;
1869     PetscScalar c[2] = {1.0, 1.0};
1870 
1871     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToCylinder));
1872     PetscCall(DMGetCoordinateDM(dm, &cdm));
1873     PetscCall(DMGetDS(cdm, &cds));
1874     PetscCall(PetscDSSetConstants(cds, 2, c));
1875   }
1876   /* Wait for coordinate creation before doing in-place modification */
1877   PetscCall(DMPlexInterpolateInPlace_Internal(dm));
1878   PetscFunctionReturn(0);
1879 }
1880 
1881 /*@
1882   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
1883 
1884   Collective
1885 
1886   Input Parameters:
1887 + comm      - The communicator for the `DM` object
1888 - periodicZ - The boundary type for the Z direction
1889 
1890   Output Parameter:
1891 . dm  - The DM object
1892 
1893   Level: beginner
1894 
1895   Note:
1896   Here is the output numbering looking from the bottom of the cylinder:
1897 .vb
1898        17-----14
1899         |     |
1900         |  2  |
1901         |     |
1902  17-----8-----7-----14
1903   |     |     |     |
1904   |  3  |  0  |  1  |
1905   |     |     |     |
1906  19-----5-----6-----13
1907         |     |
1908         |  4  |
1909         |     |
1910        19-----13
1911 
1912  and up through the top
1913 
1914        18-----16
1915         |     |
1916         |  2  |
1917         |     |
1918  18----10----11-----16
1919   |     |     |     |
1920   |  3  |  0  |  1  |
1921   |     |     |     |
1922  20-----9----12-----15
1923         |     |
1924         |  4  |
1925         |     |
1926        20-----15
1927 .ve
1928 
1929 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1930 @*/
1931 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, DMBoundaryType periodicZ, DM *dm)
1932 {
1933   PetscFunctionBegin;
1934   PetscValidPointer(dm, 3);
1935   PetscCall(DMCreate(comm, dm));
1936   PetscCall(DMSetType(*dm, DMPLEX));
1937   PetscCall(DMPlexCreateHexCylinderMesh_Internal(*dm, periodicZ));
1938   PetscFunctionReturn(0);
1939 }
1940 
1941 static PetscErrorCode DMPlexCreateWedgeCylinderMesh_Internal(DM dm, PetscInt n, PetscBool interpolate)
1942 {
1943   const PetscInt dim = 3;
1944   PetscInt       numCells, numVertices, v;
1945   PetscMPIInt    rank;
1946 
1947   PetscFunctionBegin;
1948   PetscCheck(n >= 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %" PetscInt_FMT " cannot be negative", n);
1949   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1950   PetscCall(DMSetDimension(dm, dim));
1951   /* Must create the celltype label here so that we do not automatically try to compute the types */
1952   PetscCall(DMCreateLabel(dm, "celltype"));
1953   /* Create topology */
1954   {
1955     PetscInt cone[6], c;
1956 
1957     numCells    = rank == 0 ? n : 0;
1958     numVertices = rank == 0 ? 2 * (n + 1) : 0;
1959     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1960     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
1961     PetscCall(DMSetUp(dm));
1962     for (c = 0; c < numCells; c++) {
1963       cone[0] = c + n * 1;
1964       cone[1] = (c + 1) % n + n * 1;
1965       cone[2] = 0 + 3 * n;
1966       cone[3] = c + n * 2;
1967       cone[4] = (c + 1) % n + n * 2;
1968       cone[5] = 1 + 3 * n;
1969       PetscCall(DMPlexSetCone(dm, c, cone));
1970       PetscCall(DMPlexSetCellType(dm, c, DM_POLYTOPE_TRI_PRISM_TENSOR));
1971     }
1972     PetscCall(DMPlexSymmetrize(dm));
1973     PetscCall(DMPlexStratify(dm));
1974   }
1975   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetCellType(dm, v, DM_POLYTOPE_POINT));
1976   /* Create cylinder geometry */
1977   {
1978     Vec          coordinates;
1979     PetscSection coordSection;
1980     PetscScalar *coords;
1981     PetscInt     coordSize, c;
1982 
1983     /* Build coordinates */
1984     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1985     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1986     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1987     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1988     for (v = numCells; v < numCells + numVertices; ++v) {
1989       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1990       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1991     }
1992     PetscCall(PetscSectionSetUp(coordSection));
1993     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1994     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1995     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1996     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1997     PetscCall(VecSetBlockSize(coordinates, dim));
1998     PetscCall(VecSetType(coordinates, VECSTANDARD));
1999     PetscCall(VecGetArray(coordinates, &coords));
2000     for (c = 0; c < numCells; c++) {
2001       coords[(c + 0 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
2002       coords[(c + 0 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
2003       coords[(c + 0 * n) * dim + 2] = 1.0;
2004       coords[(c + 1 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
2005       coords[(c + 1 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
2006       coords[(c + 1 * n) * dim + 2] = 0.0;
2007     }
2008     if (rank == 0) {
2009       coords[(2 * n + 0) * dim + 0] = 0.0;
2010       coords[(2 * n + 0) * dim + 1] = 0.0;
2011       coords[(2 * n + 0) * dim + 2] = 1.0;
2012       coords[(2 * n + 1) * dim + 0] = 0.0;
2013       coords[(2 * n + 1) * dim + 1] = 0.0;
2014       coords[(2 * n + 1) * dim + 2] = 0.0;
2015     }
2016     PetscCall(VecRestoreArray(coordinates, &coords));
2017     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2018     PetscCall(VecDestroy(&coordinates));
2019   }
2020   /* Interpolate */
2021   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2022   PetscFunctionReturn(0);
2023 }
2024 
2025 /*@
2026   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
2027 
2028   Collective
2029 
2030   Input Parameters:
2031 + comm - The communicator for the `DM` object
2032 . n    - The number of wedges around the origin
2033 - interpolate - Create edges and faces
2034 
2035   Output Parameter:
2036 . dm  - The `DM` object
2037 
2038   Level: beginner
2039 
2040 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2041 @*/
2042 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
2043 {
2044   PetscFunctionBegin;
2045   PetscValidPointer(dm, 4);
2046   PetscCall(DMCreate(comm, dm));
2047   PetscCall(DMSetType(*dm, DMPLEX));
2048   PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(*dm, n, interpolate));
2049   PetscFunctionReturn(0);
2050 }
2051 
2052 static inline PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2053 {
2054   PetscReal prod = 0.0;
2055   PetscInt  i;
2056   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
2057   return PetscSqrtReal(prod);
2058 }
2059 static inline PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2060 {
2061   PetscReal prod = 0.0;
2062   PetscInt  i;
2063   for (i = 0; i < dim; ++i) prod += x[i] * y[i];
2064   return prod;
2065 }
2066 
2067 /* The first constant is the sphere radius */
2068 static void snapToSphere(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
2069 {
2070   PetscReal r     = PetscRealPart(constants[0]);
2071   PetscReal norm2 = 0.0, fac;
2072   PetscInt  n     = uOff[1] - uOff[0], d;
2073 
2074   for (d = 0; d < n; ++d) norm2 += PetscSqr(PetscRealPart(u[d]));
2075   fac = r / PetscSqrtReal(norm2);
2076   for (d = 0; d < n; ++d) f0[d] = u[d] * fac;
2077 }
2078 
2079 static PetscErrorCode DMPlexCreateSphereMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, PetscReal R)
2080 {
2081   const PetscInt embedDim = dim + 1;
2082   PetscSection   coordSection;
2083   Vec            coordinates;
2084   PetscScalar   *coords;
2085   PetscReal     *coordsIn;
2086   PetscInt       numCells, numEdges, numVerts = 0, firstVertex = 0, v, firstEdge, coordSize, d, c, e;
2087   PetscMPIInt    rank;
2088 
2089   PetscFunctionBegin;
2090   PetscValidLogicalCollectiveBool(dm, simplex, 3);
2091   PetscCall(DMSetDimension(dm, dim));
2092   PetscCall(DMSetCoordinateDim(dm, dim + 1));
2093   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2094   switch (dim) {
2095   case 2:
2096     if (simplex) {
2097       const PetscReal radius    = PetscSqrtReal(1 + PETSC_PHI * PETSC_PHI) / (1.0 + PETSC_PHI);
2098       const PetscReal edgeLen   = 2.0 / (1.0 + PETSC_PHI) * (R / radius);
2099       const PetscInt  degree    = 5;
2100       PetscReal       vertex[3] = {0.0, 1.0 / (1.0 + PETSC_PHI), PETSC_PHI / (1.0 + PETSC_PHI)};
2101       PetscInt        s[3]      = {1, 1, 1};
2102       PetscInt        cone[3];
2103       PetscInt       *graph, p, i, j, k;
2104 
2105       vertex[0] *= R / radius;
2106       vertex[1] *= R / radius;
2107       vertex[2] *= R / radius;
2108       numCells    = rank == 0 ? 20 : 0;
2109       numVerts    = rank == 0 ? 12 : 0;
2110       firstVertex = numCells;
2111       /* Use icosahedron, which for a R-sphere has coordinates which are all cyclic permutations of
2112 
2113            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
2114 
2115          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2116          length is then given by 2/(1+\phi) = 2 * 0.38197 = 0.76393.
2117       */
2118       /* Construct vertices */
2119       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2120       if (rank == 0) {
2121         for (p = 0, i = 0; p < embedDim; ++p) {
2122           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2123             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2124               for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertex[(d + p) % embedDim];
2125               ++i;
2126             }
2127           }
2128         }
2129       }
2130       /* Construct graph */
2131       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2132       for (i = 0; i < numVerts; ++i) {
2133         for (j = 0, k = 0; j < numVerts; ++j) {
2134           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2135             graph[i * numVerts + j] = 1;
2136             ++k;
2137           }
2138         }
2139         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid icosahedron, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2140       }
2141       /* Build Topology */
2142       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2143       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2144       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2145       /* Cells */
2146       for (i = 0, c = 0; i < numVerts; ++i) {
2147         for (j = 0; j < i; ++j) {
2148           for (k = 0; k < j; ++k) {
2149             if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i]) {
2150               cone[0] = firstVertex + i;
2151               cone[1] = firstVertex + j;
2152               cone[2] = firstVertex + k;
2153               /* Check orientation */
2154               {
2155                 const PetscInt epsilon[3][3][3] = {
2156                   {{0, 0, 0},  {0, 0, 1},  {0, -1, 0}},
2157                   {{0, 0, -1}, {0, 0, 0},  {1, 0, 0} },
2158                   {{0, 1, 0},  {-1, 0, 0}, {0, 0, 0} }
2159                 };
2160                 PetscReal normal[3];
2161                 PetscInt  e, f;
2162 
2163                 for (d = 0; d < embedDim; ++d) {
2164                   normal[d] = 0.0;
2165                   for (e = 0; e < embedDim; ++e) {
2166                     for (f = 0; f < embedDim; ++f) normal[d] += epsilon[d][e][f] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]);
2167                   }
2168                 }
2169                 if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2170                   PetscInt tmp = cone[1];
2171                   cone[1]      = cone[2];
2172                   cone[2]      = tmp;
2173                 }
2174               }
2175               PetscCall(DMPlexSetCone(dm, c++, cone));
2176             }
2177           }
2178         }
2179       }
2180       PetscCall(DMPlexSymmetrize(dm));
2181       PetscCall(DMPlexStratify(dm));
2182       PetscCall(PetscFree(graph));
2183     } else {
2184       /*
2185         12-21--13
2186          |     |
2187         25  4  24
2188          |     |
2189   12-25--9-16--8-24--13
2190    |     |     |     |
2191   23  5 17  0 15  3  22
2192    |     |     |     |
2193   10-20--6-14--7-19--11
2194          |     |
2195         20  1  19
2196          |     |
2197         10-18--11
2198          |     |
2199         23  2  22
2200          |     |
2201         12-21--13
2202        */
2203       PetscInt cone[4], ornt[4];
2204 
2205       numCells    = rank == 0 ? 6 : 0;
2206       numEdges    = rank == 0 ? 12 : 0;
2207       numVerts    = rank == 0 ? 8 : 0;
2208       firstVertex = numCells;
2209       firstEdge   = numCells + numVerts;
2210       /* Build Topology */
2211       PetscCall(DMPlexSetChart(dm, 0, numCells + numEdges + numVerts));
2212       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 4));
2213       for (e = firstEdge; e < firstEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
2214       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2215       if (rank == 0) {
2216         /* Cell 0 */
2217         cone[0] = 14;
2218         cone[1] = 15;
2219         cone[2] = 16;
2220         cone[3] = 17;
2221         PetscCall(DMPlexSetCone(dm, 0, cone));
2222         ornt[0] = 0;
2223         ornt[1] = 0;
2224         ornt[2] = 0;
2225         ornt[3] = 0;
2226         PetscCall(DMPlexSetConeOrientation(dm, 0, ornt));
2227         /* Cell 1 */
2228         cone[0] = 18;
2229         cone[1] = 19;
2230         cone[2] = 14;
2231         cone[3] = 20;
2232         PetscCall(DMPlexSetCone(dm, 1, cone));
2233         ornt[0] = 0;
2234         ornt[1] = 0;
2235         ornt[2] = -1;
2236         ornt[3] = 0;
2237         PetscCall(DMPlexSetConeOrientation(dm, 1, ornt));
2238         /* Cell 2 */
2239         cone[0] = 21;
2240         cone[1] = 22;
2241         cone[2] = 18;
2242         cone[3] = 23;
2243         PetscCall(DMPlexSetCone(dm, 2, cone));
2244         ornt[0] = 0;
2245         ornt[1] = 0;
2246         ornt[2] = -1;
2247         ornt[3] = 0;
2248         PetscCall(DMPlexSetConeOrientation(dm, 2, ornt));
2249         /* Cell 3 */
2250         cone[0] = 19;
2251         cone[1] = 22;
2252         cone[2] = 24;
2253         cone[3] = 15;
2254         PetscCall(DMPlexSetCone(dm, 3, cone));
2255         ornt[0] = -1;
2256         ornt[1] = -1;
2257         ornt[2] = 0;
2258         ornt[3] = -1;
2259         PetscCall(DMPlexSetConeOrientation(dm, 3, ornt));
2260         /* Cell 4 */
2261         cone[0] = 16;
2262         cone[1] = 24;
2263         cone[2] = 21;
2264         cone[3] = 25;
2265         PetscCall(DMPlexSetCone(dm, 4, cone));
2266         ornt[0] = -1;
2267         ornt[1] = -1;
2268         ornt[2] = -1;
2269         ornt[3] = 0;
2270         PetscCall(DMPlexSetConeOrientation(dm, 4, ornt));
2271         /* Cell 5 */
2272         cone[0] = 20;
2273         cone[1] = 17;
2274         cone[2] = 25;
2275         cone[3] = 23;
2276         PetscCall(DMPlexSetCone(dm, 5, cone));
2277         ornt[0] = -1;
2278         ornt[1] = -1;
2279         ornt[2] = -1;
2280         ornt[3] = -1;
2281         PetscCall(DMPlexSetConeOrientation(dm, 5, ornt));
2282         /* Edges */
2283         cone[0] = 6;
2284         cone[1] = 7;
2285         PetscCall(DMPlexSetCone(dm, 14, cone));
2286         cone[0] = 7;
2287         cone[1] = 8;
2288         PetscCall(DMPlexSetCone(dm, 15, cone));
2289         cone[0] = 8;
2290         cone[1] = 9;
2291         PetscCall(DMPlexSetCone(dm, 16, cone));
2292         cone[0] = 9;
2293         cone[1] = 6;
2294         PetscCall(DMPlexSetCone(dm, 17, cone));
2295         cone[0] = 10;
2296         cone[1] = 11;
2297         PetscCall(DMPlexSetCone(dm, 18, cone));
2298         cone[0] = 11;
2299         cone[1] = 7;
2300         PetscCall(DMPlexSetCone(dm, 19, cone));
2301         cone[0] = 6;
2302         cone[1] = 10;
2303         PetscCall(DMPlexSetCone(dm, 20, cone));
2304         cone[0] = 12;
2305         cone[1] = 13;
2306         PetscCall(DMPlexSetCone(dm, 21, cone));
2307         cone[0] = 13;
2308         cone[1] = 11;
2309         PetscCall(DMPlexSetCone(dm, 22, cone));
2310         cone[0] = 10;
2311         cone[1] = 12;
2312         PetscCall(DMPlexSetCone(dm, 23, cone));
2313         cone[0] = 13;
2314         cone[1] = 8;
2315         PetscCall(DMPlexSetCone(dm, 24, cone));
2316         cone[0] = 12;
2317         cone[1] = 9;
2318         PetscCall(DMPlexSetCone(dm, 25, cone));
2319       }
2320       PetscCall(DMPlexSymmetrize(dm));
2321       PetscCall(DMPlexStratify(dm));
2322       /* Build coordinates */
2323       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2324       if (rank == 0) {
2325         coordsIn[0 * embedDim + 0] = -R;
2326         coordsIn[0 * embedDim + 1] = R;
2327         coordsIn[0 * embedDim + 2] = -R;
2328         coordsIn[1 * embedDim + 0] = R;
2329         coordsIn[1 * embedDim + 1] = R;
2330         coordsIn[1 * embedDim + 2] = -R;
2331         coordsIn[2 * embedDim + 0] = R;
2332         coordsIn[2 * embedDim + 1] = -R;
2333         coordsIn[2 * embedDim + 2] = -R;
2334         coordsIn[3 * embedDim + 0] = -R;
2335         coordsIn[3 * embedDim + 1] = -R;
2336         coordsIn[3 * embedDim + 2] = -R;
2337         coordsIn[4 * embedDim + 0] = -R;
2338         coordsIn[4 * embedDim + 1] = R;
2339         coordsIn[4 * embedDim + 2] = R;
2340         coordsIn[5 * embedDim + 0] = R;
2341         coordsIn[5 * embedDim + 1] = R;
2342         coordsIn[5 * embedDim + 2] = R;
2343         coordsIn[6 * embedDim + 0] = -R;
2344         coordsIn[6 * embedDim + 1] = -R;
2345         coordsIn[6 * embedDim + 2] = R;
2346         coordsIn[7 * embedDim + 0] = R;
2347         coordsIn[7 * embedDim + 1] = -R;
2348         coordsIn[7 * embedDim + 2] = R;
2349       }
2350     }
2351     break;
2352   case 3:
2353     if (simplex) {
2354       const PetscReal edgeLen         = 1.0 / PETSC_PHI;
2355       PetscReal       vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
2356       PetscReal       vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
2357       PetscReal       vertexC[4]      = {0.5, 0.5 * PETSC_PHI, 0.5 / PETSC_PHI, 0.0};
2358       const PetscInt  degree          = 12;
2359       PetscInt        s[4]            = {1, 1, 1};
2360       PetscInt        evenPerm[12][4] = {
2361         {0, 1, 2, 3},
2362         {0, 2, 3, 1},
2363         {0, 3, 1, 2},
2364         {1, 0, 3, 2},
2365         {1, 2, 0, 3},
2366         {1, 3, 2, 0},
2367         {2, 0, 1, 3},
2368         {2, 1, 3, 0},
2369         {2, 3, 0, 1},
2370         {3, 0, 2, 1},
2371         {3, 1, 0, 2},
2372         {3, 2, 1, 0}
2373       };
2374       PetscInt  cone[4];
2375       PetscInt *graph, p, i, j, k, l;
2376 
2377       vertexA[0] *= R;
2378       vertexA[1] *= R;
2379       vertexA[2] *= R;
2380       vertexA[3] *= R;
2381       vertexB[0] *= R;
2382       vertexB[1] *= R;
2383       vertexB[2] *= R;
2384       vertexB[3] *= R;
2385       vertexC[0] *= R;
2386       vertexC[1] *= R;
2387       vertexC[2] *= R;
2388       vertexC[3] *= R;
2389       numCells    = rank == 0 ? 600 : 0;
2390       numVerts    = rank == 0 ? 120 : 0;
2391       firstVertex = numCells;
2392       /* Use the 600-cell, which for a unit sphere has coordinates which are
2393 
2394            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
2395                (\pm 1,    0,       0,      0)  all cyclic permutations   8
2396            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
2397 
2398          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2399          length is then given by 1/\phi = 0.61803.
2400 
2401          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
2402          http://mathworld.wolfram.com/600-Cell.html
2403       */
2404       /* Construct vertices */
2405       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2406       i = 0;
2407       if (rank == 0) {
2408         for (s[0] = -1; s[0] < 2; s[0] += 2) {
2409           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2410             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2411               for (s[3] = -1; s[3] < 2; s[3] += 2) {
2412                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[d] * vertexA[d];
2413                 ++i;
2414               }
2415             }
2416           }
2417         }
2418         for (p = 0; p < embedDim; ++p) {
2419           s[1] = s[2] = s[3] = 1;
2420           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2421             for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertexB[(d + p) % embedDim];
2422             ++i;
2423           }
2424         }
2425         for (p = 0; p < 12; ++p) {
2426           s[3] = 1;
2427           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2428             for (s[1] = -1; s[1] < 2; s[1] += 2) {
2429               for (s[2] = -1; s[2] < 2; s[2] += 2) {
2430                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[evenPerm[p][d]] * vertexC[evenPerm[p][d]];
2431                 ++i;
2432               }
2433             }
2434           }
2435         }
2436       }
2437       PetscCheck(i == numVerts, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertices %" PetscInt_FMT " != %" PetscInt_FMT, i, numVerts);
2438       /* Construct graph */
2439       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2440       for (i = 0; i < numVerts; ++i) {
2441         for (j = 0, k = 0; j < numVerts; ++j) {
2442           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2443             graph[i * numVerts + j] = 1;
2444             ++k;
2445           }
2446         }
2447         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2448       }
2449       /* Build Topology */
2450       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2451       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2452       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2453       /* Cells */
2454       if (rank == 0) {
2455         for (i = 0, c = 0; i < numVerts; ++i) {
2456           for (j = 0; j < i; ++j) {
2457             for (k = 0; k < j; ++k) {
2458               for (l = 0; l < k; ++l) {
2459                 if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i] && graph[l * numVerts + i] && graph[l * numVerts + j] && graph[l * numVerts + k]) {
2460                   cone[0] = firstVertex + i;
2461                   cone[1] = firstVertex + j;
2462                   cone[2] = firstVertex + k;
2463                   cone[3] = firstVertex + l;
2464                   /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
2465                   {
2466                     const PetscInt epsilon[4][4][4][4] = {
2467                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, -1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 0, 0}, {0, 1, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 1, 0}, {0, -1, 0, 0}, {0, 0, 0, 0}}},
2468 
2469                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}}, {{0, 0, -1, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}}},
2470 
2471                       {{{0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, 0, 0}, {0, -1, 0, 0}}, {{0, 0, 0, -1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 1, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}},
2472 
2473                       {{{0, 0, 0, 0}, {0, 0, -1, 0}, {0, 1, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 1, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}}, {{0, -1, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}} }
2474                     };
2475                     PetscReal normal[4];
2476                     PetscInt  e, f, g;
2477 
2478                     for (d = 0; d < embedDim; ++d) {
2479                       normal[d] = 0.0;
2480                       for (e = 0; e < embedDim; ++e) {
2481                         for (f = 0; f < embedDim; ++f) {
2482                           for (g = 0; g < embedDim; ++g) {
2483                             normal[d] += epsilon[d][e][f][g] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]) * (coordsIn[l * embedDim + f] - coordsIn[i * embedDim + f]);
2484                           }
2485                         }
2486                       }
2487                     }
2488                     if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2489                       PetscInt tmp = cone[1];
2490                       cone[1]      = cone[2];
2491                       cone[2]      = tmp;
2492                     }
2493                   }
2494                   PetscCall(DMPlexSetCone(dm, c++, cone));
2495                 }
2496               }
2497             }
2498           }
2499         }
2500       }
2501       PetscCall(DMPlexSymmetrize(dm));
2502       PetscCall(DMPlexStratify(dm));
2503       PetscCall(PetscFree(graph));
2504     }
2505     break;
2506   default:
2507     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unsupported dimension for sphere: %" PetscInt_FMT, dim);
2508   }
2509   /* Create coordinates */
2510   PetscCall(DMGetCoordinateSection(dm, &coordSection));
2511   PetscCall(PetscSectionSetNumFields(coordSection, 1));
2512   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, embedDim));
2513   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVerts));
2514   for (v = firstVertex; v < firstVertex + numVerts; ++v) {
2515     PetscCall(PetscSectionSetDof(coordSection, v, embedDim));
2516     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, embedDim));
2517   }
2518   PetscCall(PetscSectionSetUp(coordSection));
2519   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2520   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2521   PetscCall(VecSetBlockSize(coordinates, embedDim));
2522   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2523   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2524   PetscCall(VecSetType(coordinates, VECSTANDARD));
2525   PetscCall(VecGetArray(coordinates, &coords));
2526   for (v = 0; v < numVerts; ++v)
2527     for (d = 0; d < embedDim; ++d) coords[v * embedDim + d] = coordsIn[v * embedDim + d];
2528   PetscCall(VecRestoreArray(coordinates, &coords));
2529   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2530   PetscCall(VecDestroy(&coordinates));
2531   PetscCall(PetscFree(coordsIn));
2532   {
2533     DM          cdm;
2534     PetscDS     cds;
2535     PetscScalar c = R;
2536 
2537     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToSphere));
2538     PetscCall(DMGetCoordinateDM(dm, &cdm));
2539     PetscCall(DMGetDS(cdm, &cds));
2540     PetscCall(PetscDSSetConstants(cds, 1, &c));
2541   }
2542   /* Wait for coordinate creation before doing in-place modification */
2543   if (simplex) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2544   PetscFunctionReturn(0);
2545 }
2546 
2547 typedef void (*TPSEvaluateFunc)(const PetscReal[], PetscReal *, PetscReal[], PetscReal (*)[3]);
2548 
2549 /*
2550  The Schwarz P implicit surface is
2551 
2552      f(x) = cos(x0) + cos(x1) + cos(x2) = 0
2553 */
2554 static void TPSEvaluate_SchwarzP(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2555 {
2556   PetscReal c[3] = {PetscCosReal(y[0] * PETSC_PI), PetscCosReal(y[1] * PETSC_PI), PetscCosReal(y[2] * PETSC_PI)};
2557   PetscReal g[3] = {-PetscSinReal(y[0] * PETSC_PI), -PetscSinReal(y[1] * PETSC_PI), -PetscSinReal(y[2] * PETSC_PI)};
2558   f[0]           = c[0] + c[1] + c[2];
2559   for (PetscInt i = 0; i < 3; i++) {
2560     grad[i] = PETSC_PI * g[i];
2561     for (PetscInt j = 0; j < 3; j++) hess[i][j] = (i == j) ? -PetscSqr(PETSC_PI) * c[i] : 0.;
2562   }
2563 }
2564 
2565 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2566 static PetscErrorCode TPSExtrudeNormalFunc_SchwarzP(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2567 {
2568   for (PetscInt i = 0; i < 3; i++) u[i] = -PETSC_PI * PetscSinReal(x[i] * PETSC_PI);
2569   return 0;
2570 }
2571 
2572 /*
2573  The Gyroid implicit surface is
2574 
2575  f(x,y,z) = sin(pi * x) * cos (pi * (y + 1/2))  + sin(pi * (y + 1/2)) * cos(pi * (z + 1/4)) + sin(pi * (z + 1/4)) * cos(pi * x)
2576 
2577 */
2578 static void TPSEvaluate_Gyroid(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2579 {
2580   PetscReal s[3] = {PetscSinReal(PETSC_PI * y[0]), PetscSinReal(PETSC_PI * (y[1] + .5)), PetscSinReal(PETSC_PI * (y[2] + .25))};
2581   PetscReal c[3] = {PetscCosReal(PETSC_PI * y[0]), PetscCosReal(PETSC_PI * (y[1] + .5)), PetscCosReal(PETSC_PI * (y[2] + .25))};
2582   f[0]           = s[0] * c[1] + s[1] * c[2] + s[2] * c[0];
2583   grad[0]        = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2584   grad[1]        = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2585   grad[2]        = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2586   hess[0][0]     = -PetscSqr(PETSC_PI) * (s[0] * c[1] + s[2] * c[0]);
2587   hess[0][1]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2588   hess[0][2]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2589   hess[1][0]     = -PetscSqr(PETSC_PI) * (s[1] * c[2] + s[0] * c[1]);
2590   hess[1][1]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2591   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2592   hess[2][0]     = -PetscSqr(PETSC_PI) * (s[2] * c[0] + s[1] * c[2]);
2593   hess[2][1]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2594   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2595 }
2596 
2597 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2598 static PetscErrorCode TPSExtrudeNormalFunc_Gyroid(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2599 {
2600   PetscReal s[3] = {PetscSinReal(PETSC_PI * x[0]), PetscSinReal(PETSC_PI * (x[1] + .5)), PetscSinReal(PETSC_PI * (x[2] + .25))};
2601   PetscReal c[3] = {PetscCosReal(PETSC_PI * x[0]), PetscCosReal(PETSC_PI * (x[1] + .5)), PetscCosReal(PETSC_PI * (x[2] + .25))};
2602   u[0]           = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2603   u[1]           = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2604   u[2]           = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2605   return 0;
2606 }
2607 
2608 /*
2609    We wish to solve
2610 
2611          min_y || y - x ||^2  subject to f(y) = 0
2612 
2613    Let g(y) = grad(f).  The minimization problem is equivalent to asking to satisfy
2614    f(y) = 0 and (y-x) is parallel to g(y).  We do this by using Householder QR to obtain a basis for the
2615    tangent space and ask for both components in the tangent space to be zero.
2616 
2617    Take g to be a column vector and compute the "full QR" factorization Q R = g,
2618    where Q = I - 2 n n^T is a symmetric orthogonal matrix.
2619    The first column of Q is parallel to g so the remaining two columns span the null space.
2620    Let Qn = Q[:,1:] be those remaining columns.  Then Qn Qn^T is an orthogonal projector into the tangent space.
2621    Since Q is symmetric, this is equivalent to multipyling by Q and taking the last two entries.
2622    In total, we have a system of 3 equations in 3 unknowns:
2623 
2624      f(y) = 0                       1 equation
2625      Qn^T (y - x) = 0               2 equations
2626 
2627    Here, we compute the residual and Jacobian of this system.
2628 */
2629 static void TPSNearestPointResJac(TPSEvaluateFunc feval, const PetscScalar x[], const PetscScalar y[], PetscScalar res[], PetscScalar J[])
2630 {
2631   PetscReal yreal[3] = {PetscRealPart(y[0]), PetscRealPart(y[1]), PetscRealPart(y[2])};
2632   PetscReal d[3]     = {PetscRealPart(y[0] - x[0]), PetscRealPart(y[1] - x[1]), PetscRealPart(y[2] - x[2])};
2633   PetscReal f, grad[3], n[3], norm, norm_y[3], nd, nd_y[3], sign;
2634   PetscReal n_y[3][3] = {
2635     {0, 0, 0},
2636     {0, 0, 0},
2637     {0, 0, 0}
2638   };
2639 
2640   feval(yreal, &f, grad, n_y);
2641 
2642   for (PetscInt i = 0; i < 3; i++) n[i] = grad[i];
2643   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2644   for (PetscInt i = 0; i < 3; i++) norm_y[i] = 1. / norm * n[i] * n_y[i][i];
2645 
2646   // Define the Householder reflector
2647   sign = n[0] >= 0 ? 1. : -1.;
2648   n[0] += norm * sign;
2649   for (PetscInt i = 0; i < 3; i++) n_y[0][i] += norm_y[i] * sign;
2650 
2651   norm      = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2652   norm_y[0] = 1. / norm * (n[0] * n_y[0][0]);
2653   norm_y[1] = 1. / norm * (n[0] * n_y[0][1] + n[1] * n_y[1][1]);
2654   norm_y[2] = 1. / norm * (n[0] * n_y[0][2] + n[2] * n_y[2][2]);
2655 
2656   for (PetscInt i = 0; i < 3; i++) {
2657     n[i] /= norm;
2658     for (PetscInt j = 0; j < 3; j++) {
2659       // note that n[i] is n_old[i]/norm when executing the code below
2660       n_y[i][j] = n_y[i][j] / norm - n[i] / norm * norm_y[j];
2661     }
2662   }
2663 
2664   nd = n[0] * d[0] + n[1] * d[1] + n[2] * d[2];
2665   for (PetscInt i = 0; i < 3; i++) nd_y[i] = n[i] + n_y[0][i] * d[0] + n_y[1][i] * d[1] + n_y[2][i] * d[2];
2666 
2667   res[0] = f;
2668   res[1] = d[1] - 2 * n[1] * nd;
2669   res[2] = d[2] - 2 * n[2] * nd;
2670   // J[j][i] is J_{ij} (column major)
2671   for (PetscInt j = 0; j < 3; j++) {
2672     J[0 + j * 3] = grad[j];
2673     J[1 + j * 3] = (j == 1) * 1. - 2 * (n_y[1][j] * nd + n[1] * nd_y[j]);
2674     J[2 + j * 3] = (j == 2) * 1. - 2 * (n_y[2][j] * nd + n[2] * nd_y[j]);
2675   }
2676 }
2677 
2678 /*
2679    Project x to the nearest point on the implicit surface using Newton's method.
2680 */
2681 static PetscErrorCode TPSNearestPoint(TPSEvaluateFunc feval, PetscScalar x[])
2682 {
2683   PetscScalar y[3] = {x[0], x[1], x[2]}; // Initial guess
2684 
2685   PetscFunctionBegin;
2686   for (PetscInt iter = 0; iter < 10; iter++) {
2687     PetscScalar res[3], J[9];
2688     PetscReal   resnorm;
2689     TPSNearestPointResJac(feval, x, y, res, J);
2690     resnorm = PetscSqrtReal(PetscSqr(PetscRealPart(res[0])) + PetscSqr(PetscRealPart(res[1])) + PetscSqr(PetscRealPart(res[2])));
2691     if (0) { // Turn on this monitor if you need to confirm quadratic convergence
2692       PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%" PetscInt_FMT "] res [%g %g %g]\n", iter, (double)PetscRealPart(res[0]), (double)PetscRealPart(res[1]), (double)PetscRealPart(res[2])));
2693     }
2694     if (resnorm < PETSC_SMALL) break;
2695 
2696     // Take the Newton step
2697     PetscCall(PetscKernel_A_gets_inverse_A_3(J, 0., PETSC_FALSE, NULL));
2698     PetscKernel_v_gets_v_minus_A_times_w_3(y, J, res);
2699   }
2700   for (PetscInt i = 0; i < 3; i++) x[i] = y[i];
2701   PetscFunctionReturn(0);
2702 }
2703 
2704 const char *const DMPlexTPSTypes[] = {"SCHWARZ_P", "GYROID", "DMPlexTPSType", "DMPLEX_TPS_", NULL};
2705 
2706 static PetscErrorCode DMPlexCreateTPSMesh_Internal(DM dm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness)
2707 {
2708   PetscMPIInt rank;
2709   PetscInt    topoDim = 2, spaceDim = 3, numFaces = 0, numVertices = 0, numEdges = 0;
2710   PetscInt(*edges)[2] = NULL, *edgeSets = NULL;
2711   PetscInt            *cells_flat = NULL;
2712   PetscReal           *vtxCoords  = NULL;
2713   TPSEvaluateFunc      evalFunc   = NULL;
2714   PetscSimplePointFunc normalFunc = NULL;
2715   DMLabel              label;
2716 
2717   PetscFunctionBegin;
2718   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2719   PetscCheck((layers != 0) ^ (thickness == 0.), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Layers %" PetscInt_FMT " must be nonzero iff thickness %g is nonzero", layers, (double)thickness);
2720   switch (tpstype) {
2721   case DMPLEX_TPS_SCHWARZ_P:
2722     PetscCheck(!periodic || (periodic[0] == DM_BOUNDARY_NONE && periodic[1] == DM_BOUNDARY_NONE && periodic[2] == DM_BOUNDARY_NONE), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Schwarz P does not support periodic meshes");
2723     if (rank == 0) {
2724       PetscInt(*cells)[6][4][4] = NULL; // [junction, junction-face, cell, conn]
2725       PetscInt  Njunctions = 0, Ncuts = 0, Npipes[3], vcount;
2726       PetscReal L = 1;
2727 
2728       Npipes[0]   = (extent[0] + 1) * extent[1] * extent[2];
2729       Npipes[1]   = extent[0] * (extent[1] + 1) * extent[2];
2730       Npipes[2]   = extent[0] * extent[1] * (extent[2] + 1);
2731       Njunctions  = extent[0] * extent[1] * extent[2];
2732       Ncuts       = 2 * (extent[0] * extent[1] + extent[1] * extent[2] + extent[2] * extent[0]);
2733       numVertices = 4 * (Npipes[0] + Npipes[1] + Npipes[2]) + 8 * Njunctions;
2734       PetscCall(PetscMalloc1(3 * numVertices, &vtxCoords));
2735       PetscCall(PetscMalloc1(Njunctions, &cells));
2736       PetscCall(PetscMalloc1(Ncuts * 4, &edges));
2737       PetscCall(PetscMalloc1(Ncuts * 4, &edgeSets));
2738       // x-normal pipes
2739       vcount = 0;
2740       for (PetscInt i = 0; i < extent[0] + 1; i++) {
2741         for (PetscInt j = 0; j < extent[1]; j++) {
2742           for (PetscInt k = 0; k < extent[2]; k++) {
2743             for (PetscInt l = 0; l < 4; l++) {
2744               vtxCoords[vcount++] = (2 * i - 1) * L;
2745               vtxCoords[vcount++] = 2 * j * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2746               vtxCoords[vcount++] = 2 * k * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2747             }
2748           }
2749         }
2750       }
2751       // y-normal pipes
2752       for (PetscInt i = 0; i < extent[0]; i++) {
2753         for (PetscInt j = 0; j < extent[1] + 1; j++) {
2754           for (PetscInt k = 0; k < extent[2]; k++) {
2755             for (PetscInt l = 0; l < 4; l++) {
2756               vtxCoords[vcount++] = 2 * i * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2757               vtxCoords[vcount++] = (2 * j - 1) * L;
2758               vtxCoords[vcount++] = 2 * k * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2759             }
2760           }
2761         }
2762       }
2763       // z-normal pipes
2764       for (PetscInt i = 0; i < extent[0]; i++) {
2765         for (PetscInt j = 0; j < extent[1]; j++) {
2766           for (PetscInt k = 0; k < extent[2] + 1; k++) {
2767             for (PetscInt l = 0; l < 4; l++) {
2768               vtxCoords[vcount++] = 2 * i * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2769               vtxCoords[vcount++] = 2 * j * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2770               vtxCoords[vcount++] = (2 * k - 1) * L;
2771             }
2772           }
2773         }
2774       }
2775       // junctions
2776       for (PetscInt i = 0; i < extent[0]; i++) {
2777         for (PetscInt j = 0; j < extent[1]; j++) {
2778           for (PetscInt k = 0; k < extent[2]; k++) {
2779             const PetscInt J = (i * extent[1] + j) * extent[2] + k, Jvoff = (Npipes[0] + Npipes[1] + Npipes[2]) * 4 + J * 8;
2780             PetscCheck(vcount / 3 == Jvoff, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected vertex count");
2781             for (PetscInt ii = 0; ii < 2; ii++) {
2782               for (PetscInt jj = 0; jj < 2; jj++) {
2783                 for (PetscInt kk = 0; kk < 2; kk++) {
2784                   double Ls           = (1 - sqrt(2) / 4) * L;
2785                   vtxCoords[vcount++] = 2 * i * L + (2 * ii - 1) * Ls;
2786                   vtxCoords[vcount++] = 2 * j * L + (2 * jj - 1) * Ls;
2787                   vtxCoords[vcount++] = 2 * k * L + (2 * kk - 1) * Ls;
2788                 }
2789               }
2790             }
2791             const PetscInt jfaces[3][2][4] = {
2792               {{3, 1, 0, 2}, {7, 5, 4, 6}}, // x-aligned
2793               {{5, 4, 0, 1}, {7, 6, 2, 3}}, // y-aligned
2794               {{6, 2, 0, 4}, {7, 3, 1, 5}}  // z-aligned
2795             };
2796             const PetscInt pipe_lo[3] = {// vertex numbers of pipes
2797                                          ((i * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + Npipes[0] + Npipes[1]) * 4};
2798             const PetscInt pipe_hi[3] = {// vertex numbers of pipes
2799                                          (((i + 1) * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j + 1) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + 1 + Npipes[0] + Npipes[1]) * 4};
2800             for (PetscInt dir = 0; dir < 3; dir++) { // x,y,z
2801               const PetscInt ijk[3] = {i, j, k};
2802               for (PetscInt l = 0; l < 4; l++) { // rotations
2803                 cells[J][dir * 2 + 0][l][0] = pipe_lo[dir] + l;
2804                 cells[J][dir * 2 + 0][l][1] = Jvoff + jfaces[dir][0][l];
2805                 cells[J][dir * 2 + 0][l][2] = Jvoff + jfaces[dir][0][(l - 1 + 4) % 4];
2806                 cells[J][dir * 2 + 0][l][3] = pipe_lo[dir] + (l - 1 + 4) % 4;
2807                 cells[J][dir * 2 + 1][l][0] = Jvoff + jfaces[dir][1][l];
2808                 cells[J][dir * 2 + 1][l][1] = pipe_hi[dir] + l;
2809                 cells[J][dir * 2 + 1][l][2] = pipe_hi[dir] + (l - 1 + 4) % 4;
2810                 cells[J][dir * 2 + 1][l][3] = Jvoff + jfaces[dir][1][(l - 1 + 4) % 4];
2811                 if (ijk[dir] == 0) {
2812                   edges[numEdges][0] = pipe_lo[dir] + l;
2813                   edges[numEdges][1] = pipe_lo[dir] + (l + 1) % 4;
2814                   edgeSets[numEdges] = dir * 2 + 1;
2815                   numEdges++;
2816                 }
2817                 if (ijk[dir] + 1 == extent[dir]) {
2818                   edges[numEdges][0] = pipe_hi[dir] + l;
2819                   edges[numEdges][1] = pipe_hi[dir] + (l + 1) % 4;
2820                   edgeSets[numEdges] = dir * 2 + 2;
2821                   numEdges++;
2822                 }
2823               }
2824             }
2825           }
2826         }
2827       }
2828       PetscCheck(numEdges == Ncuts * 4, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Edge count %" PetscInt_FMT " incompatible with number of cuts %" PetscInt_FMT, numEdges, Ncuts);
2829       numFaces   = 24 * Njunctions;
2830       cells_flat = cells[0][0][0];
2831     }
2832     evalFunc   = TPSEvaluate_SchwarzP;
2833     normalFunc = TPSExtrudeNormalFunc_SchwarzP;
2834     break;
2835   case DMPLEX_TPS_GYROID:
2836     if (rank == 0) {
2837       // This is a coarse mesh approximation of the gyroid shifted to being the zero of the level set
2838       //
2839       //     sin(pi*x)*cos(pi*(y+1/2)) + sin(pi*(y+1/2))*cos(pi*(z+1/4)) + sin(pi*(z+1/4))*cos(x)
2840       //
2841       // on the cell [0,2]^3.
2842       //
2843       // Think about dividing that cell into four columns, and focus on the column [0,1]x[0,1]x[0,2].
2844       // If you looked at the gyroid in that column at different slices of z you would see that it kind of spins
2845       // like a boomerang:
2846       //
2847       //     z = 0          z = 1/4        z = 1/2        z = 3/4     //
2848       //     -----          -------        -------        -------     //
2849       //                                                              //
2850       //     +       +      +       +      +       +      +   \   +   //
2851       //      \                                   /            \      //
2852       //       \            `-_   _-'            /              }     //
2853       //        *-_            `-'            _-'              /      //
2854       //     +     `-+      +       +      +-'     +      +   /   +   //
2855       //                                                              //
2856       //                                                              //
2857       //     z = 1          z = 5/4        z = 3/2        z = 7/4     //
2858       //     -----          -------        -------        -------     //
2859       //                                                              //
2860       //     +-_     +      +       +      +     _-+      +   /   +   //
2861       //        `-_            _-_            _-`            /        //
2862       //           \        _-'   `-_        /              {         //
2863       //            \                       /                \        //
2864       //     +       +      +       +      +       +      +   \   +   //
2865       //
2866       //
2867       // This course mesh approximates each of these slices by two line segments,
2868       // and then connects the segments in consecutive layers with quadrilateral faces.
2869       // All of the end points of the segments are multiples of 1/4 except for the
2870       // point * in the picture for z = 0 above and the similar points in other layers.
2871       // That point is at (gamma, gamma, 0), where gamma is calculated below.
2872       //
2873       // The column  [1,2]x[1,2]x[0,2] looks the same as this column;
2874       // The columns [1,2]x[0,1]x[0,2] and [0,1]x[1,2]x[0,2] are mirror images.
2875       //
2876       // As for how this method turned into the names given to the vertices:
2877       // that was not systematic, it was just the way it worked out in my handwritten notes.
2878 
2879       PetscInt facesPerBlock = 64;
2880       PetscInt vertsPerBlock = 56;
2881       PetscInt extentPlus[3];
2882       PetscInt numBlocks, numBlocksPlus;
2883       const PetscInt A = 0, B = 1, C = 2, D = 3, E = 4, F = 5, G = 6, H = 7, II = 8, J = 9, K = 10, L = 11, M = 12, N = 13, O = 14, P = 15, Q = 16, R = 17, S = 18, T = 19, U = 20, V = 21, W = 22, X = 23, Y = 24, Z = 25, Ap = 26, Bp = 27, Cp = 28, Dp = 29, Ep = 30, Fp = 31, Gp = 32, Hp = 33, Ip = 34, Jp = 35, Kp = 36, Lp = 37, Mp = 38, Np = 39, Op = 40, Pp = 41, Qp = 42, Rp = 43, Sp = 44, Tp = 45, Up = 46, Vp = 47, Wp = 48, Xp = 49, Yp = 50, Zp = 51, Aq = 52, Bq = 53, Cq = 54, Dq = 55;
2884       const PetscInt pattern[64][4] = {
2885   /* face to vertex within the coarse discretization of a single gyroid block */
2886   /* layer 0 */
2887         {A,           C,           K,           G          },
2888         {C,           B,           II,          K          },
2889         {D,           A,           H,           L          },
2890         {B + 56 * 1,  D,           L,           J          },
2891         {E,           B + 56 * 1,  J,           N          },
2892         {A + 56 * 2,  E,           N,           H + 56 * 2 },
2893         {F,           A + 56 * 2,  G + 56 * 2,  M          },
2894         {B,           F,           M,           II         },
2895  /* layer 1 */
2896         {G,           K,           Q,           O          },
2897         {K,           II,          P,           Q          },
2898         {L,           H,           O + 56 * 1,  R          },
2899         {J,           L,           R,           P          },
2900         {N,           J,           P,           S          },
2901         {H + 56 * 2,  N,           S,           O + 56 * 3 },
2902         {M,           G + 56 * 2,  O + 56 * 2,  T          },
2903         {II,          M,           T,           P          },
2904  /* layer 2 */
2905         {O,           Q,           Y,           U          },
2906         {Q,           P,           W,           Y          },
2907         {R,           O + 56 * 1,  U + 56 * 1,  Ap         },
2908         {P,           R,           Ap,          W          },
2909         {S,           P,           X,           Bp         },
2910         {O + 56 * 3,  S,           Bp,          V + 56 * 1 },
2911         {T,           O + 56 * 2,  V,           Z          },
2912         {P,           T,           Z,           X          },
2913  /* layer 3 */
2914         {U,           Y,           Ep,          Dp         },
2915         {Y,           W,           Cp,          Ep         },
2916         {Ap,          U + 56 * 1,  Dp + 56 * 1, Gp         },
2917         {W,           Ap,          Gp,          Cp         },
2918         {Bp,          X,           Cp + 56 * 2, Fp         },
2919         {V + 56 * 1,  Bp,          Fp,          Dp + 56 * 1},
2920         {Z,           V,           Dp,          Hp         },
2921         {X,           Z,           Hp,          Cp + 56 * 2},
2922  /* layer 4 */
2923         {Dp,          Ep,          Mp,          Kp         },
2924         {Ep,          Cp,          Ip,          Mp         },
2925         {Gp,          Dp + 56 * 1, Lp,          Np         },
2926         {Cp,          Gp,          Np,          Jp         },
2927         {Fp,          Cp + 56 * 2, Jp + 56 * 2, Pp         },
2928         {Dp + 56 * 1, Fp,          Pp,          Lp         },
2929         {Hp,          Dp,          Kp,          Op         },
2930         {Cp + 56 * 2, Hp,          Op,          Ip + 56 * 2},
2931  /* layer 5 */
2932         {Kp,          Mp,          Sp,          Rp         },
2933         {Mp,          Ip,          Qp,          Sp         },
2934         {Np,          Lp,          Rp,          Tp         },
2935         {Jp,          Np,          Tp,          Qp + 56 * 1},
2936         {Pp,          Jp + 56 * 2, Qp + 56 * 3, Up         },
2937         {Lp,          Pp,          Up,          Rp         },
2938         {Op,          Kp,          Rp,          Vp         },
2939         {Ip + 56 * 2, Op,          Vp,          Qp + 56 * 2},
2940  /* layer 6 */
2941         {Rp,          Sp,          Aq,          Yp         },
2942         {Sp,          Qp,          Wp,          Aq         },
2943         {Tp,          Rp,          Yp,          Cq         },
2944         {Qp + 56 * 1, Tp,          Cq,          Wp + 56 * 1},
2945         {Up,          Qp + 56 * 3, Xp + 56 * 1, Dq         },
2946         {Rp,          Up,          Dq,          Zp         },
2947         {Vp,          Rp,          Zp,          Bq         },
2948         {Qp + 56 * 2, Vp,          Bq,          Xp         },
2949  /* layer 7 (the top is the periodic image of the bottom of layer 0) */
2950         {Yp,          Aq,          C + 56 * 4,  A + 56 * 4 },
2951         {Aq,          Wp,          B + 56 * 4,  C + 56 * 4 },
2952         {Cq,          Yp,          A + 56 * 4,  D + 56 * 4 },
2953         {Wp + 56 * 1, Cq,          D + 56 * 4,  B + 56 * 5 },
2954         {Dq,          Xp + 56 * 1, B + 56 * 5,  E + 56 * 4 },
2955         {Zp,          Dq,          E + 56 * 4,  A + 56 * 6 },
2956         {Bq,          Zp,          A + 56 * 6,  F + 56 * 4 },
2957         {Xp,          Bq,          F + 56 * 4,  B + 56 * 4 }
2958       };
2959       const PetscReal gamma                = PetscAcosReal((PetscSqrtReal(3.) - 1.) / PetscSqrtReal(2.)) / PETSC_PI;
2960       const PetscReal patternCoords[56][3] = {
2961         {1.,        0.,        0.  }, /* A  */
2962         {0.,        1.,        0.  }, /* B  */
2963         {gamma,     gamma,     0.  }, /* C  */
2964         {1 + gamma, 1 - gamma, 0.  }, /* D  */
2965         {2 - gamma, 2 - gamma, 0.  }, /* E  */
2966         {1 - gamma, 1 + gamma, 0.  }, /* F  */
2967 
2968         {.5,        0,         .25 }, /* G  */
2969         {1.5,       0.,        .25 }, /* H  */
2970         {.5,        1.,        .25 }, /* II */
2971         {1.5,       1.,        .25 }, /* J  */
2972         {.25,       .5,        .25 }, /* K  */
2973         {1.25,      .5,        .25 }, /* L  */
2974         {.75,       1.5,       .25 }, /* M  */
2975         {1.75,      1.5,       .25 }, /* N  */
2976 
2977         {0.,        0.,        .5  }, /* O  */
2978         {1.,        1.,        .5  }, /* P  */
2979         {gamma,     1 - gamma, .5  }, /* Q  */
2980         {1 + gamma, gamma,     .5  }, /* R  */
2981         {2 - gamma, 1 + gamma, .5  }, /* S  */
2982         {1 - gamma, 2 - gamma, .5  }, /* T  */
2983 
2984         {0.,        .5,        .75 }, /* U  */
2985         {0.,        1.5,       .75 }, /* V  */
2986         {1.,        .5,        .75 }, /* W  */
2987         {1.,        1.5,       .75 }, /* X  */
2988         {.5,        .75,       .75 }, /* Y  */
2989         {.5,        1.75,      .75 }, /* Z  */
2990         {1.5,       .25,       .75 }, /* Ap */
2991         {1.5,       1.25,      .75 }, /* Bp */
2992 
2993         {1.,        0.,        1.  }, /* Cp */
2994         {0.,        1.,        1.  }, /* Dp */
2995         {1 - gamma, 1 - gamma, 1.  }, /* Ep */
2996         {1 + gamma, 1 + gamma, 1.  }, /* Fp */
2997         {2 - gamma, gamma,     1.  }, /* Gp */
2998         {gamma,     2 - gamma, 1.  }, /* Hp */
2999 
3000         {.5,        0.,        1.25}, /* Ip */
3001         {1.5,       0.,        1.25}, /* Jp */
3002         {.5,        1.,        1.25}, /* Kp */
3003         {1.5,       1.,        1.25}, /* Lp */
3004         {.75,       .5,        1.25}, /* Mp */
3005         {1.75,      .5,        1.25}, /* Np */
3006         {.25,       1.5,       1.25}, /* Op */
3007         {1.25,      1.5,       1.25}, /* Pp */
3008 
3009         {0.,        0.,        1.5 }, /* Qp */
3010         {1.,        1.,        1.5 }, /* Rp */
3011         {1 - gamma, gamma,     1.5 }, /* Sp */
3012         {2 - gamma, 1 - gamma, 1.5 }, /* Tp */
3013         {1 + gamma, 2 - gamma, 1.5 }, /* Up */
3014         {gamma,     1 + gamma, 1.5 }, /* Vp */
3015 
3016         {0.,        .5,        1.75}, /* Wp */
3017         {0.,        1.5,       1.75}, /* Xp */
3018         {1.,        .5,        1.75}, /* Yp */
3019         {1.,        1.5,       1.75}, /* Zp */
3020         {.5,        .25,       1.75}, /* Aq */
3021         {.5,        1.25,      1.75}, /* Bq */
3022         {1.5,       .75,       1.75}, /* Cq */
3023         {1.5,       1.75,      1.75}, /* Dq */
3024       };
3025       PetscInt(*cells)[64][4] = NULL;
3026       PetscBool *seen;
3027       PetscInt  *vertToTrueVert;
3028       PetscInt   count;
3029 
3030       for (PetscInt i = 0; i < 3; i++) extentPlus[i] = extent[i] + 1;
3031       numBlocks = 1;
3032       for (PetscInt i = 0; i < 3; i++) numBlocks *= extent[i];
3033       numBlocksPlus = 1;
3034       for (PetscInt i = 0; i < 3; i++) numBlocksPlus *= extentPlus[i];
3035       numFaces = numBlocks * facesPerBlock;
3036       PetscCall(PetscMalloc1(numBlocks, &cells));
3037       PetscCall(PetscCalloc1(numBlocksPlus * vertsPerBlock, &seen));
3038       for (PetscInt k = 0; k < extent[2]; k++) {
3039         for (PetscInt j = 0; j < extent[1]; j++) {
3040           for (PetscInt i = 0; i < extent[0]; i++) {
3041             for (PetscInt f = 0; f < facesPerBlock; f++) {
3042               for (PetscInt v = 0; v < 4; v++) {
3043                 PetscInt vertRaw     = pattern[f][v];
3044                 PetscInt blockidx    = vertRaw / 56;
3045                 PetscInt patternvert = vertRaw % 56;
3046                 PetscInt xplus       = (blockidx & 1);
3047                 PetscInt yplus       = (blockidx & 2) >> 1;
3048                 PetscInt zplus       = (blockidx & 4) >> 2;
3049                 PetscInt zcoord      = (periodic && periodic[2] == DM_BOUNDARY_PERIODIC) ? ((k + zplus) % extent[2]) : (k + zplus);
3050                 PetscInt ycoord      = (periodic && periodic[1] == DM_BOUNDARY_PERIODIC) ? ((j + yplus) % extent[1]) : (j + yplus);
3051                 PetscInt xcoord      = (periodic && periodic[0] == DM_BOUNDARY_PERIODIC) ? ((i + xplus) % extent[0]) : (i + xplus);
3052                 PetscInt vert        = ((zcoord * extentPlus[1] + ycoord) * extentPlus[0] + xcoord) * 56 + patternvert;
3053 
3054                 cells[(k * extent[1] + j) * extent[0] + i][f][v] = vert;
3055                 seen[vert]                                       = PETSC_TRUE;
3056               }
3057             }
3058           }
3059         }
3060       }
3061       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++)
3062         if (seen[i]) numVertices++;
3063       count = 0;
3064       PetscCall(PetscMalloc1(numBlocksPlus * vertsPerBlock, &vertToTrueVert));
3065       PetscCall(PetscMalloc1(numVertices * 3, &vtxCoords));
3066       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) vertToTrueVert[i] = -1;
3067       for (PetscInt k = 0; k < extentPlus[2]; k++) {
3068         for (PetscInt j = 0; j < extentPlus[1]; j++) {
3069           for (PetscInt i = 0; i < extentPlus[0]; i++) {
3070             for (PetscInt v = 0; v < vertsPerBlock; v++) {
3071               PetscInt vIdx = ((k * extentPlus[1] + j) * extentPlus[0] + i) * vertsPerBlock + v;
3072 
3073               if (seen[vIdx]) {
3074                 PetscInt thisVert;
3075 
3076                 vertToTrueVert[vIdx] = thisVert = count++;
3077 
3078                 for (PetscInt d = 0; d < 3; d++) vtxCoords[3 * thisVert + d] = patternCoords[v][d];
3079                 vtxCoords[3 * thisVert + 0] += i * 2;
3080                 vtxCoords[3 * thisVert + 1] += j * 2;
3081                 vtxCoords[3 * thisVert + 2] += k * 2;
3082               }
3083             }
3084           }
3085         }
3086       }
3087       for (PetscInt i = 0; i < numBlocks; i++) {
3088         for (PetscInt f = 0; f < facesPerBlock; f++) {
3089           for (PetscInt v = 0; v < 4; v++) cells[i][f][v] = vertToTrueVert[cells[i][f][v]];
3090         }
3091       }
3092       PetscCall(PetscFree(vertToTrueVert));
3093       PetscCall(PetscFree(seen));
3094       cells_flat = cells[0][0];
3095       numEdges   = 0;
3096       for (PetscInt i = 0; i < numFaces; i++) {
3097         for (PetscInt e = 0; e < 4; e++) {
3098           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3099           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3100 
3101           for (PetscInt d = 0; d < 3; d++) {
3102             if (!periodic || periodic[0] != DM_BOUNDARY_PERIODIC) {
3103               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) numEdges++;
3104               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) numEdges++;
3105             }
3106           }
3107         }
3108       }
3109       PetscCall(PetscMalloc1(numEdges, &edges));
3110       PetscCall(PetscMalloc1(numEdges, &edgeSets));
3111       for (PetscInt edge = 0, i = 0; i < numFaces; i++) {
3112         for (PetscInt e = 0; e < 4; e++) {
3113           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3114           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3115 
3116           for (PetscInt d = 0; d < 3; d++) {
3117             if (!periodic || periodic[d] != DM_BOUNDARY_PERIODIC) {
3118               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) {
3119                 edges[edge][0]   = ev[0];
3120                 edges[edge][1]   = ev[1];
3121                 edgeSets[edge++] = 2 * d;
3122               }
3123               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) {
3124                 edges[edge][0]   = ev[0];
3125                 edges[edge][1]   = ev[1];
3126                 edgeSets[edge++] = 2 * d + 1;
3127               }
3128             }
3129           }
3130         }
3131       }
3132     }
3133     evalFunc   = TPSEvaluate_Gyroid;
3134     normalFunc = TPSExtrudeNormalFunc_Gyroid;
3135     break;
3136   }
3137 
3138   PetscCall(DMSetDimension(dm, topoDim));
3139   if (rank == 0) PetscCall(DMPlexBuildFromCellList(dm, numFaces, numVertices, 4, cells_flat));
3140   else PetscCall(DMPlexBuildFromCellList(dm, 0, 0, 0, NULL));
3141   PetscCall(PetscFree(cells_flat));
3142   {
3143     DM idm;
3144     PetscCall(DMPlexInterpolate(dm, &idm));
3145     PetscCall(DMPlexReplace_Internal(dm, &idm));
3146   }
3147   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, vtxCoords));
3148   else PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, NULL));
3149   PetscCall(PetscFree(vtxCoords));
3150 
3151   PetscCall(DMCreateLabel(dm, "Face Sets"));
3152   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3153   for (PetscInt e = 0; e < numEdges; e++) {
3154     PetscInt        njoin;
3155     const PetscInt *join, verts[] = {numFaces + edges[e][0], numFaces + edges[e][1]};
3156     PetscCall(DMPlexGetJoin(dm, 2, verts, &njoin, &join));
3157     PetscCheck(njoin == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected unique join of vertices %" PetscInt_FMT " and %" PetscInt_FMT, edges[e][0], edges[e][1]);
3158     PetscCall(DMLabelSetValue(label, join[0], edgeSets[e]));
3159     PetscCall(DMPlexRestoreJoin(dm, 2, verts, &njoin, &join));
3160   }
3161   PetscCall(PetscFree(edges));
3162   PetscCall(PetscFree(edgeSets));
3163   if (tps_distribute) {
3164     DM               pdm = NULL;
3165     PetscPartitioner part;
3166 
3167     PetscCall(DMPlexGetPartitioner(dm, &part));
3168     PetscCall(PetscPartitionerSetFromOptions(part));
3169     PetscCall(DMPlexDistribute(dm, 0, NULL, &pdm));
3170     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
3171     // Do not auto-distribute again
3172     PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
3173   }
3174 
3175   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3176   for (PetscInt refine = 0; refine < refinements; refine++) {
3177     PetscInt     m;
3178     DM           dmf;
3179     Vec          X;
3180     PetscScalar *x;
3181     PetscCall(DMRefine(dm, MPI_COMM_NULL, &dmf));
3182     PetscCall(DMPlexReplace_Internal(dm, &dmf));
3183 
3184     PetscCall(DMGetCoordinatesLocal(dm, &X));
3185     PetscCall(VecGetLocalSize(X, &m));
3186     PetscCall(VecGetArray(X, &x));
3187     for (PetscInt i = 0; i < m; i += 3) PetscCall(TPSNearestPoint(evalFunc, &x[i]));
3188     PetscCall(VecRestoreArray(X, &x));
3189   }
3190 
3191   // Face Sets has already been propagated to new vertices during refinement; this propagates to the initial vertices.
3192   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3193   PetscCall(DMPlexLabelComplete(dm, label));
3194 
3195   if (thickness > 0) {
3196     DM              edm, cdm, ecdm;
3197     DMPlexTransform tr;
3198     const char     *prefix;
3199     PetscOptions    options;
3200     // Code from DMPlexExtrude
3201     PetscCall(DMPlexTransformCreate(PetscObjectComm((PetscObject)dm), &tr));
3202     PetscCall(DMPlexTransformSetDM(tr, dm));
3203     PetscCall(DMPlexTransformSetType(tr, DMPLEXEXTRUDE));
3204     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
3205     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)tr, prefix));
3206     PetscCall(PetscObjectGetOptions((PetscObject)dm, &options));
3207     PetscCall(PetscObjectSetOptions((PetscObject)tr, options));
3208     PetscCall(DMPlexTransformExtrudeSetLayers(tr, layers));
3209     PetscCall(DMPlexTransformExtrudeSetThickness(tr, thickness));
3210     PetscCall(DMPlexTransformExtrudeSetTensor(tr, PETSC_FALSE));
3211     PetscCall(DMPlexTransformExtrudeSetSymmetric(tr, PETSC_TRUE));
3212     PetscCall(DMPlexTransformExtrudeSetNormalFunction(tr, normalFunc));
3213     PetscCall(DMPlexTransformSetFromOptions(tr));
3214     PetscCall(PetscObjectSetOptions((PetscObject)tr, NULL));
3215     PetscCall(DMPlexTransformSetUp(tr));
3216     PetscCall(PetscObjectViewFromOptions((PetscObject)tr, NULL, "-dm_plex_tps_transform_view"));
3217     PetscCall(DMPlexTransformApply(tr, dm, &edm));
3218     PetscCall(DMCopyDisc(dm, edm));
3219     PetscCall(DMGetCoordinateDM(dm, &cdm));
3220     PetscCall(DMGetCoordinateDM(edm, &ecdm));
3221     PetscCall(DMCopyDisc(cdm, ecdm));
3222     PetscCall(DMPlexTransformCreateDiscLabels(tr, edm));
3223     PetscCall(DMPlexTransformDestroy(&tr));
3224     if (edm) {
3225       ((DM_Plex *)edm->data)->printFEM    = ((DM_Plex *)dm->data)->printFEM;
3226       ((DM_Plex *)edm->data)->printL2     = ((DM_Plex *)dm->data)->printL2;
3227       ((DM_Plex *)edm->data)->printLocate = ((DM_Plex *)dm->data)->printLocate;
3228     }
3229     PetscCall(DMPlexReplace_Internal(dm, &edm));
3230   }
3231   PetscFunctionReturn(0);
3232 }
3233 
3234 /*@
3235   DMPlexCreateTPSMesh - Create a distributed, interpolated mesh of a triply-periodic surface
3236 
3237   Collective
3238 
3239   Input Parameters:
3240 + comm   - The communicator for the `DM` object
3241 . tpstype - Type of triply-periodic surface
3242 . extent - Array of length 3 containing number of periods in each direction
3243 . periodic - array of length 3 with periodicity, or NULL for non-periodic
3244 . tps_distribute - Distribute 2D manifold mesh prior to refinement and extrusion (more scalable)
3245 . refinements - Number of factor-of-2 refinements of 2D manifold mesh
3246 . layers - Number of cell layers extruded in normal direction
3247 - thickness - Thickness in normal direction
3248 
3249   Output Parameter:
3250 . dm  - The `DM` object
3251 
3252   Level: beginner
3253 
3254   Notes:
3255   This meshes the surface of the Schwarz P or Gyroid surfaces.  Schwarz P is is the simplest member of the triply-periodic minimal surfaces.
3256   https://en.wikipedia.org/wiki/Schwarz_minimal_surface#Schwarz_P_(%22Primitive%22) and can be cut with "clean" boundaries.
3257   The Gyroid (https://en.wikipedia.org/wiki/Gyroid) is another triply-periodic minimal surface with applications in additive manufacturing; it is much more difficult to "cut" since there are no planes of symmetry.
3258   Our implementation creates a very coarse mesh of the surface and refines (by 4-way splitting) as many times as requested.
3259   On each refinement, all vertices are projected to their nearest point on the surface.
3260   This projection could readily be extended to related surfaces.
3261 
3262   The face (edge) sets for the Schwarz P surface are numbered 1(-x), 2(+x), 3(-y), 4(+y), 5(-z), 6(+z).
3263   When the mesh is refined, "Face Sets" contain the new vertices (created during refinement).  Use DMPlexLabelComplete() to propagate to coarse-level vertices.
3264 
3265   Developer Note:
3266   The Gyroid mesh does not currently mark boundary sets.
3267 
3268   References:
3269 . * - Maskery et al, Insights into the mechanical properties of several triply periodic minimal surface lattice structures made by polymer additive manufacturing, 2017.
3270     https://doi.org/10.1016/j.polymer.2017.11.049
3271 
3272 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMSetType()`, `DMCreate()`
3273 @*/
3274 PetscErrorCode DMPlexCreateTPSMesh(MPI_Comm comm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness, DM *dm)
3275 {
3276   PetscFunctionBegin;
3277   PetscCall(DMCreate(comm, dm));
3278   PetscCall(DMSetType(*dm, DMPLEX));
3279   PetscCall(DMPlexCreateTPSMesh_Internal(*dm, tpstype, extent, periodic, tps_distribute, refinements, layers, thickness));
3280   PetscFunctionReturn(0);
3281 }
3282 
3283 /*@
3284   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
3285 
3286   Collective
3287 
3288   Input Parameters:
3289 + comm    - The communicator for the `DM` object
3290 . dim     - The dimension
3291 . simplex - Use simplices, or tensor product cells
3292 - R       - The radius
3293 
3294   Output Parameter:
3295 . dm  - The `DM` object
3296 
3297   Level: beginner
3298 
3299 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBallMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3300 @*/
3301 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscReal R, DM *dm)
3302 {
3303   PetscFunctionBegin;
3304   PetscValidPointer(dm, 5);
3305   PetscCall(DMCreate(comm, dm));
3306   PetscCall(DMSetType(*dm, DMPLEX));
3307   PetscCall(DMPlexCreateSphereMesh_Internal(*dm, dim, simplex, R));
3308   PetscFunctionReturn(0);
3309 }
3310 
3311 static PetscErrorCode DMPlexCreateBallMesh_Internal(DM dm, PetscInt dim, PetscReal R)
3312 {
3313   DM      sdm, vol;
3314   DMLabel bdlabel;
3315 
3316   PetscFunctionBegin;
3317   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
3318   PetscCall(DMSetType(sdm, DMPLEX));
3319   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sdm, "bd_"));
3320   PetscCall(DMPlexCreateSphereMesh_Internal(sdm, dim - 1, PETSC_TRUE, R));
3321   PetscCall(DMSetFromOptions(sdm));
3322   PetscCall(DMViewFromOptions(sdm, NULL, "-dm_view"));
3323   PetscCall(DMPlexGenerate(sdm, NULL, PETSC_TRUE, &vol));
3324   PetscCall(DMDestroy(&sdm));
3325   PetscCall(DMPlexReplace_Internal(dm, &vol));
3326   PetscCall(DMCreateLabel(dm, "marker"));
3327   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
3328   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
3329   PetscCall(DMPlexLabelComplete(dm, bdlabel));
3330   PetscFunctionReturn(0);
3331 }
3332 
3333 /*@
3334   DMPlexCreateBallMesh - Creates a simplex mesh on the d-dimensional ball, B^d.
3335 
3336   Collective
3337 
3338   Input Parameters:
3339 + comm  - The communicator for the `DM` object
3340 . dim   - The dimension
3341 - R     - The radius
3342 
3343   Output Parameter:
3344 . dm  - The `DM` object
3345 
3346   Options Database Key:
3347 - bd_dm_refine - This will refine the surface mesh preserving the sphere geometry
3348 
3349   Level: beginner
3350 
3351 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3352 @*/
3353 PetscErrorCode DMPlexCreateBallMesh(MPI_Comm comm, PetscInt dim, PetscReal R, DM *dm)
3354 {
3355   PetscFunctionBegin;
3356   PetscCall(DMCreate(comm, dm));
3357   PetscCall(DMSetType(*dm, DMPLEX));
3358   PetscCall(DMPlexCreateBallMesh_Internal(*dm, dim, R));
3359   PetscFunctionReturn(0);
3360 }
3361 
3362 static PetscErrorCode DMPlexCreateReferenceCell_Internal(DM rdm, DMPolytopeType ct)
3363 {
3364   PetscFunctionBegin;
3365   switch (ct) {
3366   case DM_POLYTOPE_POINT: {
3367     PetscInt    numPoints[1]        = {1};
3368     PetscInt    coneSize[1]         = {0};
3369     PetscInt    cones[1]            = {0};
3370     PetscInt    coneOrientations[1] = {0};
3371     PetscScalar vertexCoords[1]     = {0.0};
3372 
3373     PetscCall(DMSetDimension(rdm, 0));
3374     PetscCall(DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3375   } break;
3376   case DM_POLYTOPE_SEGMENT: {
3377     PetscInt    numPoints[2]        = {2, 1};
3378     PetscInt    coneSize[3]         = {2, 0, 0};
3379     PetscInt    cones[2]            = {1, 2};
3380     PetscInt    coneOrientations[2] = {0, 0};
3381     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3382 
3383     PetscCall(DMSetDimension(rdm, 1));
3384     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3385   } break;
3386   case DM_POLYTOPE_POINT_PRISM_TENSOR: {
3387     PetscInt    numPoints[2]        = {2, 1};
3388     PetscInt    coneSize[3]         = {2, 0, 0};
3389     PetscInt    cones[2]            = {1, 2};
3390     PetscInt    coneOrientations[2] = {0, 0};
3391     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3392 
3393     PetscCall(DMSetDimension(rdm, 1));
3394     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3395   } break;
3396   case DM_POLYTOPE_TRIANGLE: {
3397     PetscInt    numPoints[2]        = {3, 1};
3398     PetscInt    coneSize[4]         = {3, 0, 0, 0};
3399     PetscInt    cones[3]            = {1, 2, 3};
3400     PetscInt    coneOrientations[3] = {0, 0, 0};
3401     PetscScalar vertexCoords[6]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0};
3402 
3403     PetscCall(DMSetDimension(rdm, 2));
3404     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3405   } break;
3406   case DM_POLYTOPE_QUADRILATERAL: {
3407     PetscInt    numPoints[2]        = {4, 1};
3408     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3409     PetscInt    cones[4]            = {1, 2, 3, 4};
3410     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3411     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0};
3412 
3413     PetscCall(DMSetDimension(rdm, 2));
3414     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3415   } break;
3416   case DM_POLYTOPE_SEG_PRISM_TENSOR: {
3417     PetscInt    numPoints[2]        = {4, 1};
3418     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3419     PetscInt    cones[4]            = {1, 2, 3, 4};
3420     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3421     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0};
3422 
3423     PetscCall(DMSetDimension(rdm, 2));
3424     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3425   } break;
3426   case DM_POLYTOPE_TETRAHEDRON: {
3427     PetscInt    numPoints[2]        = {4, 1};
3428     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3429     PetscInt    cones[4]            = {1, 2, 3, 4};
3430     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3431     PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0};
3432 
3433     PetscCall(DMSetDimension(rdm, 3));
3434     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3435   } break;
3436   case DM_POLYTOPE_HEXAHEDRON: {
3437     PetscInt    numPoints[2]        = {8, 1};
3438     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3439     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3440     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3441     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3442 
3443     PetscCall(DMSetDimension(rdm, 3));
3444     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3445   } break;
3446   case DM_POLYTOPE_TRI_PRISM: {
3447     PetscInt    numPoints[2]        = {6, 1};
3448     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3449     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3450     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3451     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3452 
3453     PetscCall(DMSetDimension(rdm, 3));
3454     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3455   } break;
3456   case DM_POLYTOPE_TRI_PRISM_TENSOR: {
3457     PetscInt    numPoints[2]        = {6, 1};
3458     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3459     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3460     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3461     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3462 
3463     PetscCall(DMSetDimension(rdm, 3));
3464     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3465   } break;
3466   case DM_POLYTOPE_QUAD_PRISM_TENSOR: {
3467     PetscInt    numPoints[2]        = {8, 1};
3468     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3469     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3470     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3471     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3472 
3473     PetscCall(DMSetDimension(rdm, 3));
3474     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3475   } break;
3476   case DM_POLYTOPE_PYRAMID: {
3477     PetscInt    numPoints[2]        = {5, 1};
3478     PetscInt    coneSize[6]         = {5, 0, 0, 0, 0, 0};
3479     PetscInt    cones[5]            = {1, 2, 3, 4, 5};
3480     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3481     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, 0.0, 0.0, 1.0};
3482 
3483     PetscCall(DMSetDimension(rdm, 3));
3484     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3485   } break;
3486   default:
3487     SETERRQ(PetscObjectComm((PetscObject)rdm), PETSC_ERR_ARG_WRONG, "Cannot create reference cell for cell type %s", DMPolytopeTypes[ct]);
3488   }
3489   {
3490     PetscInt Nv, v;
3491 
3492     /* Must create the celltype label here so that we do not automatically try to compute the types */
3493     PetscCall(DMCreateLabel(rdm, "celltype"));
3494     PetscCall(DMPlexSetCellType(rdm, 0, ct));
3495     PetscCall(DMPlexGetChart(rdm, NULL, &Nv));
3496     for (v = 1; v < Nv; ++v) PetscCall(DMPlexSetCellType(rdm, v, DM_POLYTOPE_POINT));
3497   }
3498   PetscCall(DMPlexInterpolateInPlace_Internal(rdm));
3499   PetscCall(PetscObjectSetName((PetscObject)rdm, DMPolytopeTypes[ct]));
3500   PetscFunctionReturn(0);
3501 }
3502 
3503 /*@
3504   DMPlexCreateReferenceCell - Create a `DMPLEX` with the appropriate FEM reference cell
3505 
3506   Collective
3507 
3508   Input Parameters:
3509 + comm - The communicator
3510 - ct   - The cell type of the reference cell
3511 
3512   Output Parameter:
3513 . refdm - The reference cell
3514 
3515   Level: intermediate
3516 
3517 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateReferenceCell()`, `DMPlexCreateBoxMesh()`
3518 @*/
3519 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, DMPolytopeType ct, DM *refdm)
3520 {
3521   PetscFunctionBegin;
3522   PetscCall(DMCreate(comm, refdm));
3523   PetscCall(DMSetType(*refdm, DMPLEX));
3524   PetscCall(DMPlexCreateReferenceCell_Internal(*refdm, ct));
3525   PetscFunctionReturn(0);
3526 }
3527 
3528 static PetscErrorCode DMPlexCreateBoundaryLabel_Private(DM dm, const char name[])
3529 {
3530   DM        plex;
3531   DMLabel   label;
3532   PetscBool hasLabel;
3533 
3534   PetscFunctionBegin;
3535   PetscCall(DMHasLabel(dm, name, &hasLabel));
3536   if (hasLabel) PetscFunctionReturn(0);
3537   PetscCall(DMCreateLabel(dm, name));
3538   PetscCall(DMGetLabel(dm, name, &label));
3539   PetscCall(DMConvert(dm, DMPLEX, &plex));
3540   PetscCall(DMPlexMarkBoundaryFaces(plex, 1, label));
3541   PetscCall(DMPlexLabelComplete(plex, label));
3542   PetscCall(DMDestroy(&plex));
3543   PetscFunctionReturn(0);
3544 }
3545 
3546 /*
3547   We use the last coordinate as the radius, the inner radius is lower[dim-1] and the outer radius is upper[dim-1]. Then we map the first coordinate around the circle.
3548 
3549     (x, y) -> (r, theta) = (x[1], (x[0] - lower[0]) * 2\pi/(upper[0] - lower[0]))
3550 */
3551 static void boxToAnnulus(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
3552 {
3553   const PetscReal low = PetscRealPart(constants[0]);
3554   const PetscReal upp = PetscRealPart(constants[1]);
3555   const PetscReal r   = PetscRealPart(u[1]);
3556   const PetscReal th  = 2. * PETSC_PI * (PetscRealPart(u[0]) - low) / (upp - low);
3557 
3558   f0[0] = r * PetscCosReal(th);
3559   f0[1] = r * PetscSinReal(th);
3560 }
3561 
3562 const char *const DMPlexShapes[] = {"box", "box_surface", "ball", "sphere", "cylinder", "schwarz_p", "gyroid", "doublet", "annulus", "unknown", "DMPlexShape", "DM_SHAPE_", NULL};
3563 
3564 static PetscErrorCode DMPlexCreateFromOptions_Internal(PetscOptionItems *PetscOptionsObject, PetscBool *useCoordSpace, DM dm)
3565 {
3566   DMPlexShape    shape   = DM_SHAPE_BOX;
3567   DMPolytopeType cell    = DM_POLYTOPE_TRIANGLE;
3568   PetscInt       dim     = 2;
3569   PetscBool      simplex = PETSC_TRUE, interpolate = PETSC_TRUE, adjCone = PETSC_FALSE, adjClosure = PETSC_TRUE, refDomain = PETSC_FALSE;
3570   PetscBool      flg, flg2, fflg, bdfflg, nameflg;
3571   MPI_Comm       comm;
3572   char           filename[PETSC_MAX_PATH_LEN]   = "<unspecified>";
3573   char           bdFilename[PETSC_MAX_PATH_LEN] = "<unspecified>";
3574   char           plexname[PETSC_MAX_PATH_LEN]   = "";
3575 
3576   PetscFunctionBegin;
3577   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3578   /* TODO Turn this into a registration interface */
3579   PetscCall(PetscOptionsString("-dm_plex_filename", "File containing a mesh", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &fflg));
3580   PetscCall(PetscOptionsString("-dm_plex_boundary_filename", "File containing a mesh boundary", "DMPlexCreateFromFile", bdFilename, bdFilename, sizeof(bdFilename), &bdfflg));
3581   PetscCall(PetscOptionsString("-dm_plex_name", "Name of the mesh in the file", "DMPlexCreateFromFile", plexname, plexname, sizeof(plexname), &nameflg));
3582   PetscCall(PetscOptionsEnum("-dm_plex_cell", "Cell shape", "", DMPolytopeTypes, (PetscEnum)cell, (PetscEnum *)&cell, NULL));
3583   PetscCall(PetscOptionsBool("-dm_plex_reference_cell_domain", "Use a reference cell domain", "", refDomain, &refDomain, NULL));
3584   PetscCall(PetscOptionsEnum("-dm_plex_shape", "Shape for built-in mesh", "", DMPlexShapes, (PetscEnum)shape, (PetscEnum *)&shape, &flg));
3585   PetscCall(PetscOptionsBoundedInt("-dm_plex_dim", "Topological dimension of the mesh", "DMGetDimension", dim, &dim, &flg, 0));
3586   PetscCheck(!(dim < 0) && !(dim > 3), comm, PETSC_ERR_ARG_OUTOFRANGE, "Dimension %" PetscInt_FMT " should be in [1, 3]", dim);
3587   PetscCall(PetscOptionsBool("-dm_plex_simplex", "Mesh cell shape", "", simplex, &simplex, &flg));
3588   PetscCall(PetscOptionsBool("-dm_plex_interpolate", "Flag to create edges and faces automatically", "", interpolate, &interpolate, &flg));
3589   PetscCall(PetscOptionsBool("-dm_plex_adj_cone", "Set adjacency direction", "DMSetBasicAdjacency", adjCone, &adjCone, &flg));
3590   PetscCall(PetscOptionsBool("-dm_plex_adj_closure", "Set adjacency size", "DMSetBasicAdjacency", adjClosure, &adjClosure, &flg2));
3591   if (flg || flg2) PetscCall(DMSetBasicAdjacency(dm, adjCone, adjClosure));
3592 
3593   switch (cell) {
3594   case DM_POLYTOPE_POINT:
3595   case DM_POLYTOPE_SEGMENT:
3596   case DM_POLYTOPE_POINT_PRISM_TENSOR:
3597   case DM_POLYTOPE_TRIANGLE:
3598   case DM_POLYTOPE_QUADRILATERAL:
3599   case DM_POLYTOPE_TETRAHEDRON:
3600   case DM_POLYTOPE_HEXAHEDRON:
3601     *useCoordSpace = PETSC_TRUE;
3602     break;
3603   default:
3604     *useCoordSpace = PETSC_FALSE;
3605     break;
3606   }
3607 
3608   if (fflg) {
3609     DM dmnew;
3610 
3611     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), filename, plexname, interpolate, &dmnew));
3612     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3613     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3614   } else if (refDomain) {
3615     PetscCall(DMPlexCreateReferenceCell_Internal(dm, cell));
3616   } else if (bdfflg) {
3617     DM bdm, dmnew;
3618 
3619     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), bdFilename, plexname, interpolate, &bdm));
3620     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)bdm, "bd_"));
3621     PetscCall(DMSetFromOptions(bdm));
3622     PetscCall(DMPlexGenerate(bdm, NULL, interpolate, &dmnew));
3623     PetscCall(DMDestroy(&bdm));
3624     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3625     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3626   } else {
3627     PetscCall(PetscObjectSetName((PetscObject)dm, DMPlexShapes[shape]));
3628     switch (shape) {
3629     case DM_SHAPE_BOX:
3630     case DM_SHAPE_ANNULUS: {
3631       PetscInt       faces[3]  = {0, 0, 0};
3632       PetscReal      lower[3]  = {0, 0, 0};
3633       PetscReal      upper[3]  = {1, 1, 1};
3634       DMBoundaryType bdt[3]    = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3635       PetscBool      isAnnular = shape == DM_SHAPE_ANNULUS ? PETSC_TRUE : PETSC_FALSE;
3636       PetscInt       i, n;
3637 
3638       n = dim;
3639       for (i = 0; i < dim; ++i) faces[i] = (dim == 1 ? 1 : 4 - dim);
3640       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3641       n = 3;
3642       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3643       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3644       n = 3;
3645       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3646       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3647       n = 3;
3648       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
3649       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3650 
3651       PetscCheck(!isAnnular || dim == 2, comm, PETSC_ERR_ARG_OUTOFRANGE, "Only two dimensional annuli have been implemented");
3652       if (isAnnular)
3653         for (i = 0; i < dim - 1; ++i) bdt[i] = DM_BOUNDARY_PERIODIC;
3654 
3655       switch (cell) {
3656       case DM_POLYTOPE_TRI_PRISM_TENSOR:
3657         PetscCall(DMPlexCreateWedgeBoxMesh_Internal(dm, faces, lower, upper, bdt));
3658         if (!interpolate) {
3659           DM udm;
3660 
3661           PetscCall(DMPlexUninterpolate(dm, &udm));
3662           PetscCall(DMPlexReplace_Internal(dm, &udm));
3663         }
3664         break;
3665       default:
3666         PetscCall(DMPlexCreateBoxMesh_Internal(dm, dim, simplex, faces, lower, upper, bdt, interpolate));
3667         break;
3668       }
3669       if (isAnnular) {
3670         DM          cdm;
3671         PetscDS     cds;
3672         PetscScalar bounds[2] = {lower[0], upper[0]};
3673 
3674         // Fix coordinates for annular region
3675         PetscCall(DMSetPeriodicity(dm, NULL, NULL, NULL));
3676         PetscCall(DMSetCellCoordinatesLocal(dm, NULL));
3677         PetscCall(DMSetCellCoordinates(dm, NULL));
3678         PetscCall(DMPlexCreateCoordinateSpace(dm, 1, NULL));
3679         PetscCall(DMGetCoordinateDM(dm, &cdm));
3680         PetscCall(DMGetDS(cdm, &cds));
3681         PetscCall(PetscDSSetConstants(cds, 2, bounds));
3682         PetscCall(DMPlexRemapGeometry(dm, 0.0, boxToAnnulus));
3683       }
3684     } break;
3685     case DM_SHAPE_BOX_SURFACE: {
3686       PetscInt  faces[3] = {0, 0, 0};
3687       PetscReal lower[3] = {0, 0, 0};
3688       PetscReal upper[3] = {1, 1, 1};
3689       PetscInt  i, n;
3690 
3691       n = dim + 1;
3692       for (i = 0; i < dim + 1; ++i) faces[i] = (dim + 1 == 1 ? 1 : 4 - (dim + 1));
3693       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3694       n = 3;
3695       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3696       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3697       n = 3;
3698       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3699       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3700       PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(dm, dim + 1, faces, lower, upper, interpolate));
3701     } break;
3702     case DM_SHAPE_SPHERE: {
3703       PetscReal R = 1.0;
3704 
3705       PetscCall(PetscOptionsReal("-dm_plex_sphere_radius", "Radius of the sphere", "", R, &R, &flg));
3706       PetscCall(DMPlexCreateSphereMesh_Internal(dm, dim, simplex, R));
3707     } break;
3708     case DM_SHAPE_BALL: {
3709       PetscReal R = 1.0;
3710 
3711       PetscCall(PetscOptionsReal("-dm_plex_ball_radius", "Radius of the ball", "", R, &R, &flg));
3712       PetscCall(DMPlexCreateBallMesh_Internal(dm, dim, R));
3713     } break;
3714     case DM_SHAPE_CYLINDER: {
3715       DMBoundaryType bdt = DM_BOUNDARY_NONE;
3716       PetscInt       Nw  = 6;
3717 
3718       PetscCall(PetscOptionsEnum("-dm_plex_cylinder_bd", "Boundary type in the z direction", "", DMBoundaryTypes, (PetscEnum)bdt, (PetscEnum *)&bdt, NULL));
3719       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_wedges", "Number of wedges around the cylinder", "", Nw, &Nw, NULL));
3720       switch (cell) {
3721       case DM_POLYTOPE_TRI_PRISM_TENSOR:
3722         PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(dm, Nw, interpolate));
3723         break;
3724       default:
3725         PetscCall(DMPlexCreateHexCylinderMesh_Internal(dm, bdt));
3726         break;
3727       }
3728     } break;
3729     case DM_SHAPE_SCHWARZ_P: // fallthrough
3730     case DM_SHAPE_GYROID: {
3731       PetscInt       extent[3] = {1, 1, 1}, refine = 0, layers = 0, three;
3732       PetscReal      thickness   = 0.;
3733       DMBoundaryType periodic[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3734       DMPlexTPSType  tps_type    = shape == DM_SHAPE_SCHWARZ_P ? DMPLEX_TPS_SCHWARZ_P : DMPLEX_TPS_GYROID;
3735       PetscBool      tps_distribute;
3736       PetscCall(PetscOptionsIntArray("-dm_plex_tps_extent", "Number of replicas for each of three dimensions", NULL, extent, (three = 3, &three), NULL));
3737       PetscCall(PetscOptionsInt("-dm_plex_tps_refine", "Number of refinements", NULL, refine, &refine, NULL));
3738       PetscCall(PetscOptionsEnumArray("-dm_plex_tps_periodic", "Periodicity in each of three dimensions", NULL, DMBoundaryTypes, (PetscEnum *)periodic, (three = 3, &three), NULL));
3739       PetscCall(PetscOptionsInt("-dm_plex_tps_layers", "Number of layers in volumetric extrusion (or zero to not extrude)", NULL, layers, &layers, NULL));
3740       PetscCall(PetscOptionsReal("-dm_plex_tps_thickness", "Thickness of volumetric extrusion", NULL, thickness, &thickness, NULL));
3741       PetscCall(DMPlexDistributeGetDefault(dm, &tps_distribute));
3742       PetscCall(PetscOptionsBool("-dm_plex_tps_distribute", "Distribute the 2D mesh prior to refinement and extrusion", NULL, tps_distribute, &tps_distribute, NULL));
3743       PetscCall(DMPlexCreateTPSMesh_Internal(dm, tps_type, extent, periodic, tps_distribute, refine, layers, thickness));
3744     } break;
3745     case DM_SHAPE_DOUBLET: {
3746       DM        dmnew;
3747       PetscReal rl = 0.0;
3748 
3749       PetscCall(PetscOptionsReal("-dm_plex_doublet_refinementlimit", "Refinement limit", NULL, rl, &rl, NULL));
3750       PetscCall(DMPlexCreateDoublet(PetscObjectComm((PetscObject)dm), dim, simplex, interpolate, rl, &dmnew));
3751       PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3752       PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3753     } break;
3754     default:
3755       SETERRQ(comm, PETSC_ERR_SUP, "Domain shape %s is unsupported", DMPlexShapes[shape]);
3756     }
3757   }
3758   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3759   if (!((PetscObject)dm)->name && nameflg) PetscCall(PetscObjectSetName((PetscObject)dm, plexname));
3760   PetscFunctionReturn(0);
3761 }
3762 
3763 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
3764 {
3765   DM_Plex  *mesh = (DM_Plex *)dm->data;
3766   PetscBool flg, flg2;
3767   char      bdLabel[PETSC_MAX_PATH_LEN];
3768 
3769   PetscFunctionBegin;
3770   /* Handle viewing */
3771   PetscCall(PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL));
3772   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fem", "Debug output level all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL, 0));
3773   PetscCall(PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL));
3774   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL, 0));
3775   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_locate", "Debug output level all point location computations", "DMLocatePoints", 0, &mesh->printLocate, NULL, 0));
3776   PetscCall(DMMonitorSetFromOptions(dm, "-dm_plex_monitor_throughput", "Monitor the simulation throughput", "DMPlexMonitorThroughput", DMPlexMonitorThroughput, NULL, &flg));
3777   if (flg) PetscCall(PetscLogDefaultBegin());
3778   /* Labeling */
3779   PetscCall(PetscOptionsString("-dm_plex_boundary_label", "Label to mark the mesh boundary", "", bdLabel, bdLabel, sizeof(bdLabel), &flg));
3780   if (flg) PetscCall(DMPlexCreateBoundaryLabel_Private(dm, bdLabel));
3781   /* Point Location */
3782   PetscCall(PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL));
3783   /* Partitioning and distribution */
3784   PetscCall(PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL));
3785   /* Generation and remeshing */
3786   PetscCall(PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL));
3787   /* Projection behavior */
3788   PetscCall(PetscOptionsBoundedInt("-dm_plex_max_projection_height", "Maximum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL, 0));
3789   PetscCall(PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL));
3790   /* Checking structure */
3791   {
3792     PetscBool all = PETSC_FALSE;
3793 
3794     PetscCall(PetscOptionsBool("-dm_plex_check_all", "Perform all basic checks", "DMPlexCheck", PETSC_FALSE, &all, NULL));
3795     if (all) {
3796       PetscCall(DMPlexCheck(dm));
3797     } else {
3798       PetscCall(PetscOptionsBool("-dm_plex_check_symmetry", "Check that the adjacency information in the mesh is symmetric", "DMPlexCheckSymmetry", PETSC_FALSE, &flg, &flg2));
3799       if (flg && flg2) PetscCall(DMPlexCheckSymmetry(dm));
3800       PetscCall(PetscOptionsBool("-dm_plex_check_skeleton", "Check that each cell has the correct number of vertices (only for homogeneous simplex or tensor meshes)", "DMPlexCheckSkeleton", PETSC_FALSE, &flg, &flg2));
3801       if (flg && flg2) PetscCall(DMPlexCheckSkeleton(dm, 0));
3802       PetscCall(PetscOptionsBool("-dm_plex_check_faces", "Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type", "DMPlexCheckFaces", PETSC_FALSE, &flg, &flg2));
3803       if (flg && flg2) PetscCall(DMPlexCheckFaces(dm, 0));
3804       PetscCall(PetscOptionsBool("-dm_plex_check_geometry", "Check that cells have positive volume", "DMPlexCheckGeometry", PETSC_FALSE, &flg, &flg2));
3805       if (flg && flg2) PetscCall(DMPlexCheckGeometry(dm));
3806       PetscCall(PetscOptionsBool("-dm_plex_check_pointsf", "Check some necessary conditions for PointSF", "DMPlexCheckPointSF", PETSC_FALSE, &flg, &flg2));
3807       if (flg && flg2) PetscCall(DMPlexCheckPointSF(dm, NULL, PETSC_FALSE));
3808       PetscCall(PetscOptionsBool("-dm_plex_check_interface_cones", "Check points on inter-partition interfaces have conforming order of cone points", "DMPlexCheckInterfaceCones", PETSC_FALSE, &flg, &flg2));
3809       if (flg && flg2) PetscCall(DMPlexCheckInterfaceCones(dm));
3810     }
3811     PetscCall(PetscOptionsBool("-dm_plex_check_cell_shape", "Check cell shape", "DMPlexCheckCellShape", PETSC_FALSE, &flg, &flg2));
3812     if (flg && flg2) PetscCall(DMPlexCheckCellShape(dm, PETSC_TRUE, PETSC_DETERMINE));
3813   }
3814   {
3815     PetscReal scale = 1.0;
3816 
3817     PetscCall(PetscOptionsReal("-dm_plex_scale", "Scale factor for mesh coordinates", "DMPlexScale", scale, &scale, &flg));
3818     if (flg) {
3819       Vec coordinates, coordinatesLocal;
3820 
3821       PetscCall(DMGetCoordinates(dm, &coordinates));
3822       PetscCall(DMGetCoordinatesLocal(dm, &coordinatesLocal));
3823       PetscCall(VecScale(coordinates, scale));
3824       PetscCall(VecScale(coordinatesLocal, scale));
3825     }
3826   }
3827   PetscCall(PetscPartitionerSetFromOptions(mesh->partitioner));
3828   PetscFunctionReturn(0);
3829 }
3830 
3831 PetscErrorCode DMSetFromOptions_Overlap_Plex(DM dm, PetscOptionItems *PetscOptionsObject, PetscInt *overlap)
3832 {
3833   PetscInt  numOvLabels = 16, numOvExLabels = 16;
3834   char     *ovLabelNames[16], *ovExLabelNames[16];
3835   PetscInt  numOvValues = 16, numOvExValues = 16, l;
3836   PetscBool flg;
3837 
3838   PetscFunctionBegin;
3839   PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", *overlap, overlap, NULL, 0));
3840   PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_labels", "List of overlap label names", "DMPlexDistribute", ovLabelNames, &numOvLabels, &flg));
3841   if (!flg) numOvLabels = 0;
3842   if (numOvLabels) {
3843     ((DM_Plex *)dm->data)->numOvLabels = numOvLabels;
3844     for (l = 0; l < numOvLabels; ++l) {
3845       PetscCall(DMGetLabel(dm, ovLabelNames[l], &((DM_Plex *)dm->data)->ovLabels[l]));
3846       PetscCheck(((DM_Plex *)dm->data)->ovLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovLabelNames[l]);
3847       PetscCall(PetscFree(ovLabelNames[l]));
3848     }
3849     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_values", "List of overlap label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovValues, &numOvValues, &flg));
3850     if (!flg) numOvValues = 0;
3851     PetscCheck(numOvLabels == numOvValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvLabels, numOvValues);
3852 
3853     PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_exclude_labels", "List of overlap exclude label names", "DMPlexDistribute", ovExLabelNames, &numOvExLabels, &flg));
3854     if (!flg) numOvExLabels = 0;
3855     ((DM_Plex *)dm->data)->numOvExLabels = numOvExLabels;
3856     for (l = 0; l < numOvExLabels; ++l) {
3857       PetscCall(DMGetLabel(dm, ovExLabelNames[l], &((DM_Plex *)dm->data)->ovExLabels[l]));
3858       PetscCheck(((DM_Plex *)dm->data)->ovExLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovExLabelNames[l]);
3859       PetscCall(PetscFree(ovExLabelNames[l]));
3860     }
3861     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_exclude_values", "List of overlap exclude label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovExValues, &numOvExValues, &flg));
3862     if (!flg) numOvExValues = 0;
3863     PetscCheck(numOvExLabels == numOvExValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of exclude labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvExLabels, numOvExValues);
3864   }
3865   PetscFunctionReturn(0);
3866 }
3867 
3868 static PetscErrorCode DMSetFromOptions_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
3869 {
3870   PetscFunctionList        ordlist;
3871   char                     oname[256];
3872   PetscReal                volume    = -1.0;
3873   PetscInt                 prerefine = 0, refine = 0, r, coarsen = 0, overlap = 0, extLayers = 0, dim;
3874   PetscBool                uniformOrig, created = PETSC_FALSE, uniform = PETSC_TRUE, distribute, interpolate = PETSC_TRUE, coordSpace = PETSC_TRUE, remap = PETSC_TRUE, ghostCells = PETSC_FALSE, isHierarchy, ignoreModel = PETSC_FALSE, flg;
3875   DMPlexReorderDefaultFlag reorder;
3876 
3877   PetscFunctionBegin;
3878   PetscOptionsHeadBegin(PetscOptionsObject, "DMPlex Options");
3879   if (dm->cloneOpts) goto non_refine;
3880   /* Handle automatic creation */
3881   PetscCall(DMGetDimension(dm, &dim));
3882   if (dim < 0) {
3883     PetscCall(DMPlexCreateFromOptions_Internal(PetscOptionsObject, &coordSpace, dm));
3884     created = PETSC_TRUE;
3885   }
3886   PetscCall(DMGetDimension(dm, &dim));
3887   /* Handle interpolation before distribution */
3888   PetscCall(PetscOptionsBool("-dm_plex_interpolate_pre", "Flag to interpolate mesh before distribution", "", interpolate, &interpolate, &flg));
3889   if (flg) {
3890     DMPlexInterpolatedFlag interpolated;
3891 
3892     PetscCall(DMPlexIsInterpolated(dm, &interpolated));
3893     if (interpolated == DMPLEX_INTERPOLATED_FULL && !interpolate) {
3894       DM udm;
3895 
3896       PetscCall(DMPlexUninterpolate(dm, &udm));
3897       PetscCall(DMPlexReplace_Internal(dm, &udm));
3898     } else if (interpolated != DMPLEX_INTERPOLATED_FULL && interpolate) {
3899       DM idm;
3900 
3901       PetscCall(DMPlexInterpolate(dm, &idm));
3902       PetscCall(DMPlexReplace_Internal(dm, &idm));
3903     }
3904   }
3905   /* Handle DMPlex refinement before distribution */
3906   PetscCall(PetscOptionsBool("-dm_refine_ignore_model", "Flag to ignore the geometry model when refining", "DMCreate", ignoreModel, &ignoreModel, &flg));
3907   if (flg) ((DM_Plex *)dm->data)->ignoreModel = ignoreModel;
3908   PetscCall(DMPlexGetRefinementUniform(dm, &uniformOrig));
3909   PetscCall(PetscOptionsBoundedInt("-dm_refine_pre", "The number of refinements before distribution", "DMCreate", prerefine, &prerefine, NULL, 0));
3910   PetscCall(PetscOptionsBool("-dm_refine_remap_pre", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
3911   PetscCall(PetscOptionsBool("-dm_refine_uniform_pre", "Flag for uniform refinement before distribution", "DMCreate", uniform, &uniform, &flg));
3912   if (flg) PetscCall(DMPlexSetRefinementUniform(dm, uniform));
3913   PetscCall(PetscOptionsReal("-dm_refine_volume_limit_pre", "The maximum cell volume after refinement before distribution", "DMCreate", volume, &volume, &flg));
3914   if (flg) {
3915     PetscCall(DMPlexSetRefinementUniform(dm, PETSC_FALSE));
3916     PetscCall(DMPlexSetRefinementLimit(dm, volume));
3917     prerefine = PetscMax(prerefine, 1);
3918   }
3919   for (r = 0; r < prerefine; ++r) {
3920     DM             rdm;
3921     PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
3922 
3923     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3924     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
3925     PetscCall(DMPlexReplace_Internal(dm, &rdm));
3926     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3927     if (coordFunc && remap) {
3928       PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
3929       ((DM_Plex *)dm->data)->coordFunc = coordFunc;
3930     }
3931   }
3932   PetscCall(DMPlexSetRefinementUniform(dm, uniformOrig));
3933   /* Handle DMPlex extrusion before distribution */
3934   PetscCall(PetscOptionsBoundedInt("-dm_extrude", "The number of layers to extrude", "", extLayers, &extLayers, NULL, 0));
3935   if (extLayers) {
3936     DM edm;
3937 
3938     PetscCall(DMExtrude(dm, extLayers, &edm));
3939     PetscCall(DMPlexReplace_Internal(dm, &edm));
3940     ((DM_Plex *)dm->data)->coordFunc = NULL;
3941     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3942     extLayers = 0;
3943     PetscCall(DMGetDimension(dm, &dim));
3944   }
3945   /* Handle DMPlex reordering before distribution */
3946   PetscCall(DMPlexReorderGetDefault(dm, &reorder));
3947   PetscCall(MatGetOrderingList(&ordlist));
3948   PetscCall(PetscStrncpy(oname, MATORDERINGNATURAL, sizeof(oname)));
3949   PetscCall(PetscOptionsFList("-dm_plex_reorder", "Set mesh reordering type", "DMPlexGetOrdering", ordlist, MATORDERINGNATURAL, oname, sizeof(oname), &flg));
3950   if (reorder == DMPLEX_REORDER_DEFAULT_TRUE || flg) {
3951     DM pdm;
3952     IS perm;
3953 
3954     PetscCall(DMPlexGetOrdering(dm, oname, NULL, &perm));
3955     PetscCall(DMPlexPermute(dm, perm, &pdm));
3956     PetscCall(ISDestroy(&perm));
3957     PetscCall(DMPlexReplace_Internal(dm, &pdm));
3958     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3959   }
3960   /* Handle DMPlex distribution */
3961   PetscCall(DMPlexDistributeGetDefault(dm, &distribute));
3962   PetscCall(PetscOptionsBool("-dm_distribute", "Flag to redistribute a mesh among processes", "DMPlexDistribute", distribute, &distribute, NULL));
3963   PetscCall(DMSetFromOptions_Overlap_Plex(dm, PetscOptionsObject, &overlap));
3964   if (distribute) {
3965     DM               pdm = NULL;
3966     PetscPartitioner part;
3967 
3968     PetscCall(DMPlexGetPartitioner(dm, &part));
3969     PetscCall(PetscPartitionerSetFromOptions(part));
3970     PetscCall(DMPlexDistribute(dm, overlap, NULL, &pdm));
3971     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
3972   }
3973   /* Create coordinate space */
3974   if (created) {
3975     DM_Plex  *mesh   = (DM_Plex *)dm->data;
3976     PetscInt  degree = 1;
3977     PetscBool flg;
3978 
3979     PetscCall(PetscOptionsBool("-dm_coord_space", "Use an FEM space for coordinates", "", coordSpace, &coordSpace, &flg));
3980     PetscCall(PetscOptionsInt("-dm_coord_petscspace_degree", "FEM degree for coordinate space", "", degree, &degree, NULL));
3981     if (coordSpace) PetscCall(DMPlexCreateCoordinateSpace(dm, degree, mesh->coordFunc));
3982     if (flg && !coordSpace) {
3983       DM           cdm;
3984       PetscDS      cds;
3985       PetscObject  obj;
3986       PetscClassId id;
3987 
3988       PetscCall(DMGetCoordinateDM(dm, &cdm));
3989       PetscCall(DMGetDS(cdm, &cds));
3990       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
3991       PetscCall(PetscObjectGetClassId(obj, &id));
3992       if (id == PETSCFE_CLASSID) {
3993         PetscContainer dummy;
3994 
3995         PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &dummy));
3996         PetscCall(PetscObjectSetName((PetscObject)dummy, "coordinates"));
3997         PetscCall(DMSetField(cdm, 0, NULL, (PetscObject)dummy));
3998         PetscCall(PetscContainerDestroy(&dummy));
3999         PetscCall(DMClearDS(cdm));
4000       }
4001       mesh->coordFunc = NULL;
4002     }
4003     PetscCall(PetscOptionsBool("-dm_sparse_localize", "Localize only necessary cells", "", dm->sparseLocalize, &dm->sparseLocalize, &flg));
4004     PetscCall(DMLocalizeCoordinates(dm));
4005   }
4006   /* Handle DMPlex refinement */
4007   remap = PETSC_TRUE;
4008   PetscCall(PetscOptionsBoundedInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL, 0));
4009   PetscCall(PetscOptionsBool("-dm_refine_remap", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
4010   PetscCall(PetscOptionsBoundedInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy, 0));
4011   if (refine) PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
4012   if (refine && isHierarchy) {
4013     DM *dms, coarseDM;
4014 
4015     PetscCall(DMGetCoarseDM(dm, &coarseDM));
4016     PetscCall(PetscObjectReference((PetscObject)coarseDM));
4017     PetscCall(PetscMalloc1(refine, &dms));
4018     PetscCall(DMRefineHierarchy(dm, refine, dms));
4019     /* Total hack since we do not pass in a pointer */
4020     PetscCall(DMPlexSwap_Static(dm, dms[refine - 1]));
4021     if (refine == 1) {
4022       PetscCall(DMSetCoarseDM(dm, dms[0]));
4023       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4024     } else {
4025       PetscCall(DMSetCoarseDM(dm, dms[refine - 2]));
4026       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4027       PetscCall(DMSetCoarseDM(dms[0], dms[refine - 1]));
4028       PetscCall(DMPlexSetRegularRefinement(dms[0], PETSC_TRUE));
4029     }
4030     PetscCall(DMSetCoarseDM(dms[refine - 1], coarseDM));
4031     PetscCall(PetscObjectDereference((PetscObject)coarseDM));
4032     /* Free DMs */
4033     for (r = 0; r < refine; ++r) {
4034       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4035       PetscCall(DMDestroy(&dms[r]));
4036     }
4037     PetscCall(PetscFree(dms));
4038   } else {
4039     for (r = 0; r < refine; ++r) {
4040       DM             rdm;
4041       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4042 
4043       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4044       PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
4045       /* Total hack since we do not pass in a pointer */
4046       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4047       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4048       if (coordFunc && remap) {
4049         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4050         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4051       }
4052     }
4053   }
4054   /* Handle DMPlex coarsening */
4055   PetscCall(PetscOptionsBoundedInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL, 0));
4056   PetscCall(PetscOptionsBoundedInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy, 0));
4057   if (coarsen && isHierarchy) {
4058     DM *dms;
4059 
4060     PetscCall(PetscMalloc1(coarsen, &dms));
4061     PetscCall(DMCoarsenHierarchy(dm, coarsen, dms));
4062     /* Free DMs */
4063     for (r = 0; r < coarsen; ++r) {
4064       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4065       PetscCall(DMDestroy(&dms[r]));
4066     }
4067     PetscCall(PetscFree(dms));
4068   } else {
4069     for (r = 0; r < coarsen; ++r) {
4070       DM             cdm;
4071       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4072 
4073       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4074       PetscCall(DMCoarsen(dm, PetscObjectComm((PetscObject)dm), &cdm));
4075       /* Total hack since we do not pass in a pointer */
4076       PetscCall(DMPlexReplace_Internal(dm, &cdm));
4077       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4078       if (coordFunc) {
4079         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4080         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4081       }
4082     }
4083   }
4084   /* Handle ghost cells */
4085   PetscCall(PetscOptionsBool("-dm_plex_create_fv_ghost_cells", "Flag to create finite volume ghost cells on the boundary", "DMCreate", ghostCells, &ghostCells, NULL));
4086   if (ghostCells) {
4087     DM   gdm;
4088     char lname[PETSC_MAX_PATH_LEN];
4089 
4090     lname[0] = '\0';
4091     PetscCall(PetscOptionsString("-dm_plex_fv_ghost_cells_label", "Label name for ghost cells boundary", "DMCreate", lname, lname, sizeof(lname), &flg));
4092     PetscCall(DMPlexConstructGhostCells(dm, flg ? lname : NULL, NULL, &gdm));
4093     PetscCall(DMPlexReplace_Internal(dm, &gdm));
4094   }
4095   /* Handle 1D order */
4096   if (reorder != DMPLEX_REORDER_DEFAULT_FALSE && dim == 1) {
4097     DM           cdm, rdm;
4098     PetscDS      cds;
4099     PetscObject  obj;
4100     PetscClassId id = PETSC_OBJECT_CLASSID;
4101     IS           perm;
4102     PetscInt     Nf;
4103     PetscBool    distributed;
4104 
4105     PetscCall(DMPlexIsDistributed(dm, &distributed));
4106     PetscCall(DMGetCoordinateDM(dm, &cdm));
4107     PetscCall(DMGetDS(cdm, &cds));
4108     PetscCall(PetscDSGetNumFields(cds, &Nf));
4109     if (Nf) {
4110       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
4111       PetscCall(PetscObjectGetClassId(obj, &id));
4112     }
4113     if (!distributed && id != PETSCFE_CLASSID) {
4114       PetscCall(DMPlexGetOrdering1D(dm, &perm));
4115       PetscCall(DMPlexPermute(dm, perm, &rdm));
4116       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4117       PetscCall(ISDestroy(&perm));
4118     }
4119   }
4120 /* Handle */
4121 non_refine:
4122   PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4123   PetscOptionsHeadEnd();
4124   PetscFunctionReturn(0);
4125 }
4126 
4127 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm, Vec *vec)
4128 {
4129   PetscFunctionBegin;
4130   PetscCall(DMCreateGlobalVector_Section_Private(dm, vec));
4131   /* PetscCall(VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM)); */
4132   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex));
4133   PetscCall(VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void))VecView_Plex_Native));
4134   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex));
4135   PetscCall(VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void))VecLoad_Plex_Native));
4136   PetscFunctionReturn(0);
4137 }
4138 
4139 static PetscErrorCode DMCreateLocalVector_Plex(DM dm, Vec *vec)
4140 {
4141   PetscFunctionBegin;
4142   PetscCall(DMCreateLocalVector_Section_Private(dm, vec));
4143   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex_Local));
4144   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex_Local));
4145   PetscFunctionReturn(0);
4146 }
4147 
4148 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
4149 {
4150   PetscInt depth, d;
4151 
4152   PetscFunctionBegin;
4153   PetscCall(DMPlexGetDepth(dm, &depth));
4154   if (depth == 1) {
4155     PetscCall(DMGetDimension(dm, &d));
4156     if (dim == 0) PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4157     else if (dim == d) PetscCall(DMPlexGetDepthStratum(dm, 1, pStart, pEnd));
4158     else {
4159       *pStart = 0;
4160       *pEnd   = 0;
4161     }
4162   } else {
4163     PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4164   }
4165   PetscFunctionReturn(0);
4166 }
4167 
4168 static PetscErrorCode DMGetNeighbors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
4169 {
4170   PetscSF            sf;
4171   PetscInt           niranks, njranks, n;
4172   const PetscMPIInt *iranks, *jranks;
4173   DM_Plex           *data = (DM_Plex *)dm->data;
4174 
4175   PetscFunctionBegin;
4176   PetscCall(DMGetPointSF(dm, &sf));
4177   if (!data->neighbors) {
4178     PetscCall(PetscSFSetUp(sf));
4179     PetscCall(PetscSFGetRootRanks(sf, &njranks, &jranks, NULL, NULL, NULL));
4180     PetscCall(PetscSFGetLeafRanks(sf, &niranks, &iranks, NULL, NULL));
4181     PetscCall(PetscMalloc1(njranks + niranks + 1, &data->neighbors));
4182     PetscCall(PetscArraycpy(data->neighbors + 1, jranks, njranks));
4183     PetscCall(PetscArraycpy(data->neighbors + njranks + 1, iranks, niranks));
4184     n = njranks + niranks;
4185     PetscCall(PetscSortRemoveDupsMPIInt(&n, data->neighbors + 1));
4186     /* The following cast should never fail: can't have more neighbors than PETSC_MPI_INT_MAX */
4187     PetscCall(PetscMPIIntCast(n, data->neighbors));
4188   }
4189   if (nranks) *nranks = data->neighbors[0];
4190   if (ranks) {
4191     if (data->neighbors[0]) *ranks = data->neighbors + 1;
4192     else *ranks = NULL;
4193   }
4194   PetscFunctionReturn(0);
4195 }
4196 
4197 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM, DM, Mat, Vec, Vec);
4198 
4199 static PetscErrorCode DMInitialize_Plex(DM dm)
4200 {
4201   PetscFunctionBegin;
4202   dm->ops->view                      = DMView_Plex;
4203   dm->ops->load                      = DMLoad_Plex;
4204   dm->ops->setfromoptions            = DMSetFromOptions_Plex;
4205   dm->ops->clone                     = DMClone_Plex;
4206   dm->ops->setup                     = DMSetUp_Plex;
4207   dm->ops->createlocalsection        = DMCreateLocalSection_Plex;
4208   dm->ops->createdefaultconstraints  = DMCreateDefaultConstraints_Plex;
4209   dm->ops->createglobalvector        = DMCreateGlobalVector_Plex;
4210   dm->ops->createlocalvector         = DMCreateLocalVector_Plex;
4211   dm->ops->getlocaltoglobalmapping   = NULL;
4212   dm->ops->createfieldis             = NULL;
4213   dm->ops->createcoordinatedm        = DMCreateCoordinateDM_Plex;
4214   dm->ops->createcoordinatefield     = DMCreateCoordinateField_Plex;
4215   dm->ops->getcoloring               = NULL;
4216   dm->ops->creatematrix              = DMCreateMatrix_Plex;
4217   dm->ops->createinterpolation       = DMCreateInterpolation_Plex;
4218   dm->ops->createmassmatrix          = DMCreateMassMatrix_Plex;
4219   dm->ops->createmassmatrixlumped    = DMCreateMassMatrixLumped_Plex;
4220   dm->ops->createinjection           = DMCreateInjection_Plex;
4221   dm->ops->refine                    = DMRefine_Plex;
4222   dm->ops->coarsen                   = DMCoarsen_Plex;
4223   dm->ops->refinehierarchy           = DMRefineHierarchy_Plex;
4224   dm->ops->coarsenhierarchy          = DMCoarsenHierarchy_Plex;
4225   dm->ops->extrude                   = DMExtrude_Plex;
4226   dm->ops->globaltolocalbegin        = NULL;
4227   dm->ops->globaltolocalend          = NULL;
4228   dm->ops->localtoglobalbegin        = NULL;
4229   dm->ops->localtoglobalend          = NULL;
4230   dm->ops->destroy                   = DMDestroy_Plex;
4231   dm->ops->createsubdm               = DMCreateSubDM_Plex;
4232   dm->ops->createsuperdm             = DMCreateSuperDM_Plex;
4233   dm->ops->getdimpoints              = DMGetDimPoints_Plex;
4234   dm->ops->locatepoints              = DMLocatePoints_Plex;
4235   dm->ops->projectfunctionlocal      = DMProjectFunctionLocal_Plex;
4236   dm->ops->projectfunctionlabellocal = DMProjectFunctionLabelLocal_Plex;
4237   dm->ops->projectfieldlocal         = DMProjectFieldLocal_Plex;
4238   dm->ops->projectfieldlabellocal    = DMProjectFieldLabelLocal_Plex;
4239   dm->ops->projectbdfieldlabellocal  = DMProjectBdFieldLabelLocal_Plex;
4240   dm->ops->computel2diff             = DMComputeL2Diff_Plex;
4241   dm->ops->computel2gradientdiff     = DMComputeL2GradientDiff_Plex;
4242   dm->ops->computel2fielddiff        = DMComputeL2FieldDiff_Plex;
4243   dm->ops->getneighbors              = DMGetNeighbors_Plex;
4244   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertBoundaryValues_C", DMPlexInsertBoundaryValues_Plex));
4245   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertTimeDerviativeBoundaryValues_C", DMPlexInsertTimeDerivativeBoundaryValues_Plex));
4246   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMSetUpGLVisViewer_C", DMSetUpGLVisViewer_Plex));
4247   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMCreateNeumannOverlap_C", DMCreateNeumannOverlap_Plex));
4248   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4249   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeGetDefault_C", DMPlexDistributeGetDefault_Plex));
4250   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeSetDefault_C", DMPlexDistributeSetDefault_Plex));
4251   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderGetDefault_C", DMPlexReorderGetDefault_Plex));
4252   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderSetDefault_C", DMPlexReorderSetDefault_Plex));
4253   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMInterpolateSolution_C", DMInterpolateSolution_Plex));
4254   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4255   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetOverlap_C", DMPlexSetOverlap_Plex));
4256   PetscFunctionReturn(0);
4257 }
4258 
4259 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
4260 {
4261   DM_Plex *mesh = (DM_Plex *)dm->data;
4262 
4263   PetscFunctionBegin;
4264   mesh->refct++;
4265   (*newdm)->data = mesh;
4266   PetscCall(PetscObjectChangeTypeName((PetscObject)*newdm, DMPLEX));
4267   PetscCall(DMInitialize_Plex(*newdm));
4268   PetscFunctionReturn(0);
4269 }
4270 
4271 /*MC
4272   DMPLEX = "plex" - A `DM` object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
4273                     In the local representation, Vecs contain all unknowns in the interior and shared boundary. This is
4274                     specified by a PetscSection object. Ownership in the global representation is determined by
4275                     ownership of the underlying `DMPLEX` points. This is specified by another `PetscSection` object.
4276 
4277   Options Database Keys:
4278 + -dm_refine_pre                     - Refine mesh before distribution
4279 + -dm_refine_uniform_pre             - Choose uniform or generator-based refinement
4280 + -dm_refine_volume_limit_pre        - Cell volume limit after pre-refinement using generator
4281 . -dm_distribute                     - Distribute mesh across processes
4282 . -dm_distribute_overlap             - Number of cells to overlap for distribution
4283 . -dm_refine                         - Refine mesh after distribution
4284 . -dm_plex_hash_location             - Use grid hashing for point location
4285 . -dm_plex_hash_box_faces <n,m,p>    - The number of divisions in each direction of the grid hash
4286 . -dm_plex_partition_balance         - Attempt to evenly divide points on partition boundary between processes
4287 . -dm_plex_remesh_bd                 - Allow changes to the boundary on remeshing
4288 . -dm_plex_max_projection_height     - Maximum mesh point height used to project locally
4289 . -dm_plex_regular_refinement        - Use special nested projection algorithm for regular refinement
4290 . -dm_plex_check_all                 - Perform all shecks below
4291 . -dm_plex_check_symmetry            - Check that the adjacency information in the mesh is symmetric
4292 . -dm_plex_check_skeleton <celltype> - Check that each cell has the correct number of vertices
4293 . -dm_plex_check_faces <celltype>    - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
4294 . -dm_plex_check_geometry            - Check that cells have positive volume
4295 . -dm_view :mesh.tex:ascii_latex     - View the mesh in LaTeX/TikZ
4296 . -dm_plex_view_scale <num>          - Scale the TikZ
4297 - -dm_plex_print_fem <num>           - View FEM assembly information, such as element vectors and matrices
4298 
4299   Level: intermediate
4300 
4301 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`, `PetscSection`
4302 M*/
4303 
4304 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
4305 {
4306   DM_Plex *mesh;
4307   PetscInt unit;
4308 
4309   PetscFunctionBegin;
4310   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4311   PetscCall(PetscNew(&mesh));
4312   dm->data = mesh;
4313 
4314   mesh->refct = 1;
4315   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection));
4316   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection));
4317   mesh->refinementUniform      = PETSC_TRUE;
4318   mesh->refinementLimit        = -1.0;
4319   mesh->distDefault            = PETSC_TRUE;
4320   mesh->reorderDefault         = DMPLEX_REORDER_DEFAULT_NOTSET;
4321   mesh->distributionName       = NULL;
4322   mesh->interpolated           = DMPLEX_INTERPOLATED_INVALID;
4323   mesh->interpolatedCollective = DMPLEX_INTERPOLATED_INVALID;
4324 
4325   PetscCall(PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner));
4326   mesh->remeshBd = PETSC_FALSE;
4327 
4328   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
4329 
4330   mesh->depthState    = -1;
4331   mesh->celltypeState = -1;
4332   mesh->printTol      = 1.0e-10;
4333 
4334   PetscCall(DMInitialize_Plex(dm));
4335   PetscFunctionReturn(0);
4336 }
4337 
4338 /*@
4339   DMPlexCreate - Creates a `DMPLEX` object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
4340 
4341   Collective
4342 
4343   Input Parameter:
4344 . comm - The communicator for the `DMPLEX` object
4345 
4346   Output Parameter:
4347 . mesh  - The `DMPLEX` object
4348 
4349   Level: beginner
4350 
4351 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`
4352 @*/
4353 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
4354 {
4355   PetscFunctionBegin;
4356   PetscValidPointer(mesh, 2);
4357   PetscCall(DMCreate(comm, mesh));
4358   PetscCall(DMSetType(*mesh, DMPLEX));
4359   PetscFunctionReturn(0);
4360 }
4361 
4362 /*@C
4363   DMPlexBuildFromCellListParallel - Build distributed `DMPLEX` topology from a list of vertices for each cell (common mesh generator output)
4364 
4365   Collective on dm
4366 
4367   Input Parameters:
4368 + dm - The `DM`
4369 . numCells - The number of cells owned by this process
4370 . numVertices - The number of vertices to be owned by this process, or `PETSC_DECIDE`
4371 . NVertices - The global number of vertices, or `PETSC_DETERMINE`
4372 . numCorners - The number of vertices for each cell
4373 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4374 
4375   Output Parameters:
4376 + vertexSF - (Optional) `PetscSF` describing complete vertex ownership
4377 - verticesAdjSaved - (Optional) vertex adjacency array
4378 
4379   Level: advanced
4380 
4381   Notes:
4382   Two triangles sharing a face
4383 .vb
4384 
4385         2
4386       / | \
4387      /  |  \
4388     /   |   \
4389    0  0 | 1  3
4390     \   |   /
4391      \  |  /
4392       \ | /
4393         1
4394 .ve
4395 would have input
4396 .vb
4397   numCells = 2, numVertices = 4
4398   cells = [0 1 2  1 3 2]
4399 .ve
4400 which would result in the `DMPLEX`
4401 .vb
4402 
4403         4
4404       / | \
4405      /  |  \
4406     /   |   \
4407    2  0 | 1  5
4408     \   |   /
4409      \  |  /
4410       \ | /
4411         3
4412 .ve
4413 
4414   Vertices are implicitly numbered consecutively 0,...,NVertices.
4415   Each rank owns a chunk of numVertices consecutive vertices.
4416   If numVertices is `PETSC_DECIDE`, PETSc will distribute them as evenly as possible using PetscLayout.
4417   If NVertices is `PETSC_DETERMINE` and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
4418   If only NVertices is `PETSC_DETERMINE`, it is computed as the sum of numVertices over all ranks.
4419 
4420   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
4421 
4422   Fortran Note:
4423   Not currently supported in Fortran.
4424 
4425 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildCoordinatesFromCellListParallel()`,
4426           `PetscSF`
4427 @*/
4428 PetscErrorCode DMPlexBuildFromCellListParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, const PetscInt cells[], PetscSF *vertexSF, PetscInt **verticesAdjSaved)
4429 {
4430   PetscSF     sfPoint;
4431   PetscLayout layout;
4432   PetscInt    numVerticesAdj, *verticesAdj, *cones, c, p;
4433 
4434   PetscFunctionBegin;
4435   PetscValidLogicalCollectiveInt(dm, NVertices, 4);
4436   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4437   /* Get/check global number of vertices */
4438   {
4439     PetscInt       NVerticesInCells, i;
4440     const PetscInt len = numCells * numCorners;
4441 
4442     /* NVerticesInCells = max(cells) + 1 */
4443     NVerticesInCells = PETSC_MIN_INT;
4444     for (i = 0; i < len; i++)
4445       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4446     ++NVerticesInCells;
4447     PetscCallMPI(MPI_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
4448 
4449     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
4450     else
4451       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
4452   }
4453   /* Count locally unique vertices */
4454   {
4455     PetscHSetI vhash;
4456     PetscInt   off = 0;
4457 
4458     PetscCall(PetscHSetICreate(&vhash));
4459     for (c = 0; c < numCells; ++c) {
4460       for (p = 0; p < numCorners; ++p) PetscCall(PetscHSetIAdd(vhash, cells[c * numCorners + p]));
4461     }
4462     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
4463     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
4464     else verticesAdj = *verticesAdjSaved;
4465     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
4466     PetscCall(PetscHSetIDestroy(&vhash));
4467     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
4468   }
4469   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
4470   /* Create cones */
4471   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
4472   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
4473   PetscCall(DMSetUp(dm));
4474   PetscCall(DMPlexGetCones(dm, &cones));
4475   for (c = 0; c < numCells; ++c) {
4476     for (p = 0; p < numCorners; ++p) {
4477       const PetscInt gv = cells[c * numCorners + p];
4478       PetscInt       lv;
4479 
4480       /* Positions within verticesAdj form 0-based local vertex numbering;
4481          we need to shift it by numCells to get correct DAG points (cells go first) */
4482       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
4483       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
4484       cones[c * numCorners + p] = lv + numCells;
4485     }
4486   }
4487   /* Build point sf */
4488   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
4489   PetscCall(PetscLayoutSetSize(layout, NVertices));
4490   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
4491   PetscCall(PetscLayoutSetBlockSize(layout, 1));
4492   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
4493   PetscCall(PetscLayoutDestroy(&layout));
4494   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
4495   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
4496   if (dm->sf) {
4497     const char *prefix;
4498 
4499     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
4500     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
4501   }
4502   PetscCall(DMSetPointSF(dm, sfPoint));
4503   PetscCall(PetscSFDestroy(&sfPoint));
4504   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)(*vertexSF), "Vertex Ownership SF"));
4505   /* Fill in the rest of the topology structure */
4506   PetscCall(DMPlexSymmetrize(dm));
4507   PetscCall(DMPlexStratify(dm));
4508   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4509   PetscFunctionReturn(0);
4510 }
4511 
4512 /*@C
4513   DMPlexBuildCoordinatesFromCellListParallel - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4514 
4515   Collective on dm
4516 
4517   Input Parameters:
4518 + dm - The `DM`
4519 . spaceDim - The spatial dimension used for coordinates
4520 . sfVert - `PetscSF` describing complete vertex ownership
4521 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4522 
4523   Level: advanced
4524 
4525   Fortran Note:
4526   Not currently supported in Fortran.
4527 
4528 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellListParallel()`
4529 @*/
4530 PetscErrorCode DMPlexBuildCoordinatesFromCellListParallel(DM dm, PetscInt spaceDim, PetscSF sfVert, const PetscReal vertexCoords[])
4531 {
4532   PetscSection coordSection;
4533   Vec          coordinates;
4534   PetscScalar *coords;
4535   PetscInt     numVertices, numVerticesAdj, coordSize, v, vStart, vEnd;
4536 
4537   PetscFunctionBegin;
4538   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4539   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
4540   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4541   PetscCall(DMSetCoordinateDim(dm, spaceDim));
4542   PetscCall(PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL));
4543   PetscCheck(vEnd - vStart == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Supplied sfVert has wrong number of leaves = %" PetscInt_FMT " != %" PetscInt_FMT " = vEnd - vStart", numVerticesAdj, vEnd - vStart);
4544   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4545   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4546   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
4547   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
4548   for (v = vStart; v < vEnd; ++v) {
4549     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
4550     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
4551   }
4552   PetscCall(PetscSectionSetUp(coordSection));
4553   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4554   PetscCall(VecCreate(PetscObjectComm((PetscObject)dm), &coordinates));
4555   PetscCall(VecSetBlockSize(coordinates, spaceDim));
4556   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4557   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4558   PetscCall(VecSetType(coordinates, VECSTANDARD));
4559   PetscCall(VecGetArray(coordinates, &coords));
4560   {
4561     MPI_Datatype coordtype;
4562 
4563     /* Need a temp buffer for coords if we have complex/single */
4564     PetscCallMPI(MPI_Type_contiguous(spaceDim, MPIU_SCALAR, &coordtype));
4565     PetscCallMPI(MPI_Type_commit(&coordtype));
4566 #if defined(PETSC_USE_COMPLEX)
4567     {
4568       PetscScalar *svertexCoords;
4569       PetscInt     i;
4570       PetscCall(PetscMalloc1(numVertices * spaceDim, &svertexCoords));
4571       for (i = 0; i < numVertices * spaceDim; i++) svertexCoords[i] = vertexCoords[i];
4572       PetscCall(PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4573       PetscCall(PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4574       PetscCall(PetscFree(svertexCoords));
4575     }
4576 #else
4577     PetscCall(PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4578     PetscCall(PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4579 #endif
4580     PetscCallMPI(MPI_Type_free(&coordtype));
4581   }
4582   PetscCall(VecRestoreArray(coordinates, &coords));
4583   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4584   PetscCall(VecDestroy(&coordinates));
4585   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4586   PetscFunctionReturn(0);
4587 }
4588 
4589 /*@
4590   DMPlexCreateFromCellListParallelPetsc - Create distributed `DMPLEX` from a list of vertices for each cell (common mesh generator output)
4591 
4592   Collective
4593 
4594   Input Parameters:
4595 + comm - The communicator
4596 . dim - The topological dimension of the mesh
4597 . numCells - The number of cells owned by this process
4598 . numVertices - The number of vertices owned by this process, or `PETSC_DECIDE`
4599 . NVertices - The global number of vertices, or `PETSC_DECIDE`
4600 . numCorners - The number of vertices for each cell
4601 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4602 . cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4603 . spaceDim - The spatial dimension used for coordinates
4604 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4605 
4606   Output Parameters:
4607 + dm - The `DM`
4608 . vertexSF - (Optional) `PetscSF` describing complete vertex ownership
4609 - verticesAdjSaved - (Optional) vertex adjacency array
4610 
4611   Level: intermediate
4612 
4613   Notes:
4614   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`,
4615   `DMPlexBuildFromCellListParallel()`, `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellListParallel()`
4616 
4617   See `DMPlexBuildFromCellListParallel()` for an example and details about the topology-related parameters.
4618 
4619   See `DMPlexBuildCoordinatesFromCellListParallel()` for details about the geometry-related parameters.
4620 
4621 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
4622 @*/
4623 PetscErrorCode DMPlexCreateFromCellListParallelPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, PetscInt **verticesAdj, DM *dm)
4624 {
4625   PetscSF sfVert;
4626 
4627   PetscFunctionBegin;
4628   PetscCall(DMCreate(comm, dm));
4629   PetscCall(DMSetType(*dm, DMPLEX));
4630   PetscValidLogicalCollectiveInt(*dm, dim, 2);
4631   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
4632   PetscCall(DMSetDimension(*dm, dim));
4633   PetscCall(DMPlexBuildFromCellListParallel(*dm, numCells, numVertices, NVertices, numCorners, cells, &sfVert, verticesAdj));
4634   if (interpolate) {
4635     DM idm;
4636 
4637     PetscCall(DMPlexInterpolate(*dm, &idm));
4638     PetscCall(DMDestroy(dm));
4639     *dm = idm;
4640   }
4641   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
4642   if (vertexSF) *vertexSF = sfVert;
4643   else PetscCall(PetscSFDestroy(&sfVert));
4644   PetscFunctionReturn(0);
4645 }
4646 
4647 /*@C
4648   DMPlexBuildFromCellList - Build `DMPLEX` topology from a list of vertices for each cell (common mesh generator output)
4649 
4650   Collective on dm
4651 
4652   Input Parameters:
4653 + dm - The `DM`
4654 . numCells - The number of cells owned by this process
4655 . numVertices - The number of vertices owned by this process, or `PETSC_DETERMINE`
4656 . numCorners - The number of vertices for each cell
4657 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4658 
4659   Level: advanced
4660 
4661   Notes:
4662   Two triangles sharing a face
4663 .vb
4664 
4665         2
4666       / | \
4667      /  |  \
4668     /   |   \
4669    0  0 | 1  3
4670     \   |   /
4671      \  |  /
4672       \ | /
4673         1
4674 .ve
4675 would have input
4676 .vb
4677   numCells = 2, numVertices = 4
4678   cells = [0 1 2  1 3 2]
4679 .ve
4680 which would result in the `DMPLEX`
4681 .vb
4682 
4683         4
4684       / | \
4685      /  |  \
4686     /   |   \
4687    2  0 | 1  5
4688     \   |   /
4689      \  |  /
4690       \ | /
4691         3
4692 .ve
4693 
4694   If numVertices is `PETSC_DETERMINE`, it is computed by PETSc as the maximum vertex index in cells + 1.
4695 
4696   Not currently supported in Fortran.
4697 
4698 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListPetsc()`
4699 @*/
4700 PetscErrorCode DMPlexBuildFromCellList(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const PetscInt cells[])
4701 {
4702   PetscInt *cones, c, p, dim;
4703 
4704   PetscFunctionBegin;
4705   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4706   PetscCall(DMGetDimension(dm, &dim));
4707   /* Get/check global number of vertices */
4708   {
4709     PetscInt       NVerticesInCells, i;
4710     const PetscInt len = numCells * numCorners;
4711 
4712     /* NVerticesInCells = max(cells) + 1 */
4713     NVerticesInCells = PETSC_MIN_INT;
4714     for (i = 0; i < len; i++)
4715       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4716     ++NVerticesInCells;
4717 
4718     if (numVertices == PETSC_DECIDE) numVertices = NVerticesInCells;
4719     else
4720       PetscCheck(numVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, numVertices, NVerticesInCells);
4721   }
4722   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
4723   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
4724   PetscCall(DMSetUp(dm));
4725   PetscCall(DMPlexGetCones(dm, &cones));
4726   for (c = 0; c < numCells; ++c) {
4727     for (p = 0; p < numCorners; ++p) cones[c * numCorners + p] = cells[c * numCorners + p] + numCells;
4728   }
4729   PetscCall(DMPlexSymmetrize(dm));
4730   PetscCall(DMPlexStratify(dm));
4731   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4732   PetscFunctionReturn(0);
4733 }
4734 
4735 /*@C
4736   DMPlexBuildCoordinatesFromCellList - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4737 
4738   Collective on dm
4739 
4740   Input Parameters:
4741 + dm - The `DM`
4742 . spaceDim - The spatial dimension used for coordinates
4743 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4744 
4745   Level: advanced
4746 
4747   Fortran Note:
4748   Not currently supported in Fortran.
4749 
4750 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellList()`
4751 @*/
4752 PetscErrorCode DMPlexBuildCoordinatesFromCellList(DM dm, PetscInt spaceDim, const PetscReal vertexCoords[])
4753 {
4754   PetscSection coordSection;
4755   Vec          coordinates;
4756   DM           cdm;
4757   PetscScalar *coords;
4758   PetscInt     v, vStart, vEnd, d;
4759 
4760   PetscFunctionBegin;
4761   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4762   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
4763   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4764   PetscCall(DMSetCoordinateDim(dm, spaceDim));
4765   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4766   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4767   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
4768   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
4769   for (v = vStart; v < vEnd; ++v) {
4770     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
4771     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
4772   }
4773   PetscCall(PetscSectionSetUp(coordSection));
4774 
4775   PetscCall(DMGetCoordinateDM(dm, &cdm));
4776   PetscCall(DMCreateLocalVector(cdm, &coordinates));
4777   PetscCall(VecSetBlockSize(coordinates, spaceDim));
4778   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4779   PetscCall(VecGetArrayWrite(coordinates, &coords));
4780   for (v = 0; v < vEnd - vStart; ++v) {
4781     for (d = 0; d < spaceDim; ++d) coords[v * spaceDim + d] = vertexCoords[v * spaceDim + d];
4782   }
4783   PetscCall(VecRestoreArrayWrite(coordinates, &coords));
4784   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4785   PetscCall(VecDestroy(&coordinates));
4786   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4787   PetscFunctionReturn(0);
4788 }
4789 
4790 /*@
4791   DMPlexCreateFromCellListPetsc - Create `DMPLEX` from a list of vertices for each cell (common mesh generator output), but only process 0 takes in the input
4792 
4793   Collective
4794 
4795   Input Parameters:
4796 + comm - The communicator
4797 . dim - The topological dimension of the mesh
4798 . numCells - The number of cells, only on process 0
4799 . numVertices - The number of vertices owned by this process, or `PETSC_DECIDE`, only on process 0
4800 . numCorners - The number of vertices for each cell, only on process 0
4801 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4802 . cells - An array of numCells*numCorners numbers, the vertices for each cell, only on process 0
4803 . spaceDim - The spatial dimension used for coordinates
4804 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex, only on process 0
4805 
4806   Output Parameter:
4807 . dm - The `DM`, which only has points on process 0
4808 
4809   Level: intermediate
4810 
4811   Notes:
4812   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`, `DMPlexBuildFromCellList()`,
4813   `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellList()`
4814 
4815   See `DMPlexBuildFromCellList()` for an example and details about the topology-related parameters.
4816   See `DMPlexBuildCoordinatesFromCellList()` for details about the geometry-related parameters.
4817   See `DMPlexCreateFromCellListParallelPetsc()` for parallel input
4818 
4819 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellList()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
4820 @*/
4821 PetscErrorCode DMPlexCreateFromCellListPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], DM *dm)
4822 {
4823   PetscMPIInt rank;
4824 
4825   PetscFunctionBegin;
4826   PetscCheck(dim, comm, PETSC_ERR_ARG_OUTOFRANGE, "This is not appropriate for 0-dimensional meshes. Consider either creating the DM using DMPlexCreateFromDAG(), by hand, or using DMSwarm.");
4827   PetscCallMPI(MPI_Comm_rank(comm, &rank));
4828   PetscCall(DMCreate(comm, dm));
4829   PetscCall(DMSetType(*dm, DMPLEX));
4830   PetscCall(DMSetDimension(*dm, dim));
4831   if (rank == 0) PetscCall(DMPlexBuildFromCellList(*dm, numCells, numVertices, numCorners, cells));
4832   else PetscCall(DMPlexBuildFromCellList(*dm, 0, 0, 0, NULL));
4833   if (interpolate) {
4834     DM idm;
4835 
4836     PetscCall(DMPlexInterpolate(*dm, &idm));
4837     PetscCall(DMDestroy(dm));
4838     *dm = idm;
4839   }
4840   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, vertexCoords));
4841   else PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, NULL));
4842   PetscFunctionReturn(0);
4843 }
4844 
4845 /*@
4846   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a DM
4847 
4848   Input Parameters:
4849 + dm - The empty DM object, usually from DMCreate() and DMSetDimension()
4850 . depth - The depth of the DAG
4851 . numPoints - Array of size depth + 1 containing the number of points at each depth
4852 . coneSize - The cone size of each point
4853 . cones - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
4854 . coneOrientations - The orientation of each cone point
4855 - vertexCoords - An array of numPoints[0]*spacedim numbers representing the coordinates of each vertex, with spacedim the value set via DMSetCoordinateDim()
4856 
4857   Output Parameter:
4858 . dm - The DM
4859 
4860   Note:
4861   Two triangles sharing a face would have input
4862 .vb
4863   depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
4864   cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
4865  vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
4866 .ve
4867 which would result in the DMPlex
4868 .vb
4869         4
4870       / | \
4871      /  |  \
4872     /   |   \
4873    2  0 | 1  5
4874     \   |   /
4875      \  |  /
4876       \ | /
4877         3
4878 .ve
4879  Notice that all points are numbered consecutively, unlike `DMPlexCreateFromCellListPetsc()`
4880 
4881   Level: advanced
4882 
4883 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
4884 @*/
4885 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
4886 {
4887   Vec          coordinates;
4888   PetscSection coordSection;
4889   PetscScalar *coords;
4890   PetscInt     coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
4891 
4892   PetscFunctionBegin;
4893   PetscCall(DMGetDimension(dm, &dim));
4894   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
4895   PetscCheck(dimEmbed >= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Embedding dimension %" PetscInt_FMT " cannot be less than intrinsic dimension %" PetscInt_FMT, dimEmbed, dim);
4896   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
4897   PetscCall(DMPlexSetChart(dm, pStart, pEnd));
4898   for (p = pStart; p < pEnd; ++p) {
4899     PetscCall(DMPlexSetConeSize(dm, p, coneSize[p - pStart]));
4900     if (firstVertex < 0 && !coneSize[p - pStart]) firstVertex = p - pStart;
4901   }
4902   PetscCheck(firstVertex >= 0 || !numPoints[0], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Expected %" PetscInt_FMT " vertices but could not find any", numPoints[0]);
4903   PetscCall(DMSetUp(dm)); /* Allocate space for cones */
4904   for (p = pStart, off = 0; p < pEnd; off += coneSize[p - pStart], ++p) {
4905     PetscCall(DMPlexSetCone(dm, p, &cones[off]));
4906     PetscCall(DMPlexSetConeOrientation(dm, p, &coneOrientations[off]));
4907   }
4908   PetscCall(DMPlexSymmetrize(dm));
4909   PetscCall(DMPlexStratify(dm));
4910   /* Build coordinates */
4911   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4912   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4913   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dimEmbed));
4914   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numPoints[0]));
4915   for (v = firstVertex; v < firstVertex + numPoints[0]; ++v) {
4916     PetscCall(PetscSectionSetDof(coordSection, v, dimEmbed));
4917     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed));
4918   }
4919   PetscCall(PetscSectionSetUp(coordSection));
4920   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4921   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
4922   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4923   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4924   PetscCall(VecSetBlockSize(coordinates, dimEmbed));
4925   PetscCall(VecSetType(coordinates, VECSTANDARD));
4926   if (vertexCoords) {
4927     PetscCall(VecGetArray(coordinates, &coords));
4928     for (v = 0; v < numPoints[0]; ++v) {
4929       PetscInt off;
4930 
4931       PetscCall(PetscSectionGetOffset(coordSection, v + firstVertex, &off));
4932       for (d = 0; d < dimEmbed; ++d) coords[off + d] = vertexCoords[v * dimEmbed + d];
4933     }
4934   }
4935   PetscCall(VecRestoreArray(coordinates, &coords));
4936   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4937   PetscCall(VecDestroy(&coordinates));
4938   PetscFunctionReturn(0);
4939 }
4940 
4941 /*@C
4942   DMPlexCreateCellVertexFromFile - Create a `DMPLEX` mesh from a simple cell-vertex file.
4943 
4944   Collective
4945 
4946 + comm        - The MPI communicator
4947 . filename    - Name of the .dat file
4948 - interpolate - Create faces and edges in the mesh
4949 
4950   Output Parameter:
4951 . dm  - The `DM` object representing the mesh
4952 
4953   Level: beginner
4954 
4955   Note:
4956   The format is the simplest possible:
4957 .vb
4958   Ne
4959   v0 v1 ... vk
4960   Nv
4961   x y z marker
4962 .ve
4963 
4964   Developer Note:
4965   Should use a `PetscViewer` not a filename
4966 
4967 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromFile()`, `DMPlexCreateMedFromFile()`, `DMPlexCreateGmsh()`, `DMPlexCreate()`
4968 @*/
4969 PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
4970 {
4971   DMLabel      marker;
4972   PetscViewer  viewer;
4973   Vec          coordinates;
4974   PetscSection coordSection;
4975   PetscScalar *coords;
4976   char         line[PETSC_MAX_PATH_LEN];
4977   PetscInt     dim = 3, cdim = 3, coordSize, v, c, d;
4978   PetscMPIInt  rank;
4979   int          snum, Nv, Nc, Ncn, Nl;
4980 
4981   PetscFunctionBegin;
4982   PetscCallMPI(MPI_Comm_rank(comm, &rank));
4983   PetscCall(PetscViewerCreate(comm, &viewer));
4984   PetscCall(PetscViewerSetType(viewer, PETSCVIEWERASCII));
4985   PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
4986   PetscCall(PetscViewerFileSetName(viewer, filename));
4987   if (rank == 0) {
4988     PetscCall(PetscViewerRead(viewer, line, 4, NULL, PETSC_STRING));
4989     snum = sscanf(line, "%d %d %d %d", &Nc, &Nv, &Ncn, &Nl);
4990     PetscCheck(snum == 4, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4991   } else {
4992     Nc = Nv = Ncn = Nl = 0;
4993   }
4994   PetscCall(DMCreate(comm, dm));
4995   PetscCall(DMSetType(*dm, DMPLEX));
4996   PetscCall(DMPlexSetChart(*dm, 0, Nc + Nv));
4997   PetscCall(DMSetDimension(*dm, dim));
4998   PetscCall(DMSetCoordinateDim(*dm, cdim));
4999   /* Read topology */
5000   if (rank == 0) {
5001     char     format[PETSC_MAX_PATH_LEN];
5002     PetscInt cone[8];
5003     int      vbuf[8], v;
5004 
5005     for (c = 0; c < Ncn; ++c) {
5006       format[c * 3 + 0] = '%';
5007       format[c * 3 + 1] = 'd';
5008       format[c * 3 + 2] = ' ';
5009     }
5010     format[Ncn * 3 - 1] = '\0';
5011     for (c = 0; c < Nc; ++c) PetscCall(DMPlexSetConeSize(*dm, c, Ncn));
5012     PetscCall(DMSetUp(*dm));
5013     for (c = 0; c < Nc; ++c) {
5014       PetscCall(PetscViewerRead(viewer, line, Ncn, NULL, PETSC_STRING));
5015       switch (Ncn) {
5016       case 2:
5017         snum = sscanf(line, format, &vbuf[0], &vbuf[1]);
5018         break;
5019       case 3:
5020         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2]);
5021         break;
5022       case 4:
5023         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3]);
5024         break;
5025       case 6:
5026         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5]);
5027         break;
5028       case 8:
5029         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);
5030         break;
5031       default:
5032         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No cell shape with %d vertices", Ncn);
5033       }
5034       PetscCheck(snum == Ncn, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5035       for (v = 0; v < Ncn; ++v) cone[v] = vbuf[v] + Nc;
5036       /* Hexahedra are inverted */
5037       if (Ncn == 8) {
5038         PetscInt tmp = cone[1];
5039         cone[1]      = cone[3];
5040         cone[3]      = tmp;
5041       }
5042       PetscCall(DMPlexSetCone(*dm, c, cone));
5043     }
5044   }
5045   PetscCall(DMPlexSymmetrize(*dm));
5046   PetscCall(DMPlexStratify(*dm));
5047   /* Read coordinates */
5048   PetscCall(DMGetCoordinateSection(*dm, &coordSection));
5049   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5050   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
5051   PetscCall(PetscSectionSetChart(coordSection, Nc, Nc + Nv));
5052   for (v = Nc; v < Nc + Nv; ++v) {
5053     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
5054     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
5055   }
5056   PetscCall(PetscSectionSetUp(coordSection));
5057   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
5058   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
5059   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5060   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
5061   PetscCall(VecSetBlockSize(coordinates, cdim));
5062   PetscCall(VecSetType(coordinates, VECSTANDARD));
5063   PetscCall(VecGetArray(coordinates, &coords));
5064   if (rank == 0) {
5065     char   format[PETSC_MAX_PATH_LEN];
5066     double x[3];
5067     int    l, val[3];
5068 
5069     if (Nl) {
5070       for (l = 0; l < Nl; ++l) {
5071         format[l * 3 + 0] = '%';
5072         format[l * 3 + 1] = 'd';
5073         format[l * 3 + 2] = ' ';
5074       }
5075       format[Nl * 3 - 1] = '\0';
5076       PetscCall(DMCreateLabel(*dm, "marker"));
5077       PetscCall(DMGetLabel(*dm, "marker", &marker));
5078     }
5079     for (v = 0; v < Nv; ++v) {
5080       PetscCall(PetscViewerRead(viewer, line, 3 + Nl, NULL, PETSC_STRING));
5081       snum = sscanf(line, "%lg %lg %lg", &x[0], &x[1], &x[2]);
5082       PetscCheck(snum == 3, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5083       switch (Nl) {
5084       case 0:
5085         snum = 0;
5086         break;
5087       case 1:
5088         snum = sscanf(line, format, &val[0]);
5089         break;
5090       case 2:
5091         snum = sscanf(line, format, &val[0], &val[1]);
5092         break;
5093       case 3:
5094         snum = sscanf(line, format, &val[0], &val[1], &val[2]);
5095         break;
5096       default:
5097         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Request support for %d labels", Nl);
5098       }
5099       PetscCheck(snum == Nl, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5100       for (d = 0; d < cdim; ++d) coords[v * cdim + d] = x[d];
5101       for (l = 0; l < Nl; ++l) PetscCall(DMLabelSetValue(marker, v + Nc, val[l]));
5102     }
5103   }
5104   PetscCall(VecRestoreArray(coordinates, &coords));
5105   PetscCall(DMSetCoordinatesLocal(*dm, coordinates));
5106   PetscCall(VecDestroy(&coordinates));
5107   PetscCall(PetscViewerDestroy(&viewer));
5108   if (interpolate) {
5109     DM      idm;
5110     DMLabel bdlabel;
5111 
5112     PetscCall(DMPlexInterpolate(*dm, &idm));
5113     PetscCall(DMDestroy(dm));
5114     *dm = idm;
5115 
5116     if (!Nl) {
5117       PetscCall(DMCreateLabel(*dm, "marker"));
5118       PetscCall(DMGetLabel(*dm, "marker", &bdlabel));
5119       PetscCall(DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel));
5120       PetscCall(DMPlexLabelComplete(*dm, bdlabel));
5121     }
5122   }
5123   PetscFunctionReturn(0);
5124 }
5125 
5126 /*@C
5127   DMPlexCreateFromFile - This takes a filename and produces a `DM`
5128 
5129   Collective
5130 
5131   Input Parameters:
5132 + comm - The communicator
5133 . filename - A file name
5134 . plexname - The object name of the resulting `DM`, also used for intra-datafile lookup by some formats
5135 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
5136 
5137   Output Parameter:
5138 . dm - The `DM`
5139 
5140   Options Database Key:
5141 . -dm_plex_create_from_hdf5_xdmf - use the `PETSC_VIEWER_HDF5_XDMF` format for reading HDF5
5142 
5143   Use -dm_plex_create_ prefix to pass options to the internal PetscViewer, e.g.
5144 $ -dm_plex_create_viewer_hdf5_collective
5145 
5146   Level: beginner
5147 
5148   Notes:
5149   Using `PETSCVIEWERHDF5` type with `PETSC_VIEWER_HDF5_PETSC` format, one can save multiple `DMPLEX`
5150   meshes in a single HDF5 file. This in turn requires one to name the `DMPLEX` object with `PetscObjectSetName()`
5151   before saving it with `DMView()` and before loading it with `DMLoad()` for identification of the mesh object.
5152   The input parameter name is thus used to name the `DMPLEX` object when `DMPlexCreateFromFile()` internally
5153   calls `DMLoad()`. Currently, name is ignored for other viewer types and/or formats.
5154 
5155 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`, `PetscObjectSetName()`, `DMView()`, `DMLoad()`
5156 @*/
5157 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], const char plexname[], PetscBool interpolate, DM *dm)
5158 {
5159   const char  extGmsh[]      = ".msh";
5160   const char  extGmsh2[]     = ".msh2";
5161   const char  extGmsh4[]     = ".msh4";
5162   const char  extCGNS[]      = ".cgns";
5163   const char  extExodus[]    = ".exo";
5164   const char  extExodus_e[]  = ".e";
5165   const char  extGenesis[]   = ".gen";
5166   const char  extFluent[]    = ".cas";
5167   const char  extHDF5[]      = ".h5";
5168   const char  extMed[]       = ".med";
5169   const char  extPLY[]       = ".ply";
5170   const char  extEGADSLite[] = ".egadslite";
5171   const char  extEGADS[]     = ".egads";
5172   const char  extIGES[]      = ".igs";
5173   const char  extSTEP[]      = ".stp";
5174   const char  extCV[]        = ".dat";
5175   size_t      len;
5176   PetscBool   isGmsh, isGmsh2, isGmsh4, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isMed, isPLY, isEGADSLite, isEGADS, isIGES, isSTEP, isCV;
5177   PetscMPIInt rank;
5178 
5179   PetscFunctionBegin;
5180   PetscValidCharPointer(filename, 2);
5181   if (plexname) PetscValidCharPointer(plexname, 3);
5182   PetscValidPointer(dm, 5);
5183   PetscCall(DMInitializePackage());
5184   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5185   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5186   PetscCall(PetscStrlen(filename, &len));
5187   PetscCheck(len, comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
5188 
5189 #define CheckExtension(extension__, is_extension__) \
5190   do { \
5191     PetscAssert(sizeof(extension__), comm, PETSC_ERR_PLIB, "Zero-size extension: %s", extension__); \
5192     /* don't count the null-terminator at the end */ \
5193     const size_t ext_len = sizeof(extension__) - 1; \
5194     if (len < ext_len) { \
5195       is_extension__ = PETSC_FALSE; \
5196     } else { \
5197       PetscCall(PetscStrncmp(filename + len - ext_len, extension__, ext_len, &is_extension__)); \
5198     } \
5199   } while (0)
5200 
5201   CheckExtension(extGmsh, isGmsh);
5202   CheckExtension(extGmsh2, isGmsh2);
5203   CheckExtension(extGmsh4, isGmsh4);
5204   CheckExtension(extCGNS, isCGNS);
5205   CheckExtension(extExodus, isExodus);
5206   if (!isExodus) CheckExtension(extExodus_e, isExodus);
5207   CheckExtension(extGenesis, isGenesis);
5208   CheckExtension(extFluent, isFluent);
5209   CheckExtension(extHDF5, isHDF5);
5210   CheckExtension(extMed, isMed);
5211   CheckExtension(extPLY, isPLY);
5212   CheckExtension(extEGADSLite, isEGADSLite);
5213   CheckExtension(extEGADS, isEGADS);
5214   CheckExtension(extIGES, isIGES);
5215   CheckExtension(extSTEP, isSTEP);
5216   CheckExtension(extCV, isCV);
5217 
5218 #undef CheckExtension
5219 
5220   if (isGmsh || isGmsh2 || isGmsh4) {
5221     PetscCall(DMPlexCreateGmshFromFile(comm, filename, interpolate, dm));
5222   } else if (isCGNS) {
5223     PetscCall(DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm));
5224   } else if (isExodus || isGenesis) {
5225     PetscCall(DMPlexCreateExodusFromFile(comm, filename, interpolate, dm));
5226   } else if (isFluent) {
5227     PetscCall(DMPlexCreateFluentFromFile(comm, filename, interpolate, dm));
5228   } else if (isHDF5) {
5229     PetscBool   load_hdf5_xdmf = PETSC_FALSE;
5230     PetscViewer viewer;
5231 
5232     /* PETSC_VIEWER_HDF5_XDMF is used if the filename ends with .xdmf.h5, or if -dm_plex_create_from_hdf5_xdmf option is present */
5233     PetscCall(PetscStrncmp(&filename[PetscMax(0, len - 8)], ".xdmf", 5, &load_hdf5_xdmf));
5234     PetscCall(PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &load_hdf5_xdmf, NULL));
5235     PetscCall(PetscViewerCreate(comm, &viewer));
5236     PetscCall(PetscViewerSetType(viewer, PETSCVIEWERHDF5));
5237     PetscCall(PetscViewerSetOptionsPrefix(viewer, "dm_plex_create_"));
5238     PetscCall(PetscViewerSetFromOptions(viewer));
5239     PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
5240     PetscCall(PetscViewerFileSetName(viewer, filename));
5241 
5242     PetscCall(DMCreate(comm, dm));
5243     PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5244     PetscCall(DMSetType(*dm, DMPLEX));
5245     if (load_hdf5_xdmf) PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF));
5246     PetscCall(DMLoad(*dm, viewer));
5247     if (load_hdf5_xdmf) PetscCall(PetscViewerPopFormat(viewer));
5248     PetscCall(PetscViewerDestroy(&viewer));
5249 
5250     if (interpolate) {
5251       DM idm;
5252 
5253       PetscCall(DMPlexInterpolate(*dm, &idm));
5254       PetscCall(DMDestroy(dm));
5255       *dm = idm;
5256     }
5257   } else if (isMed) {
5258     PetscCall(DMPlexCreateMedFromFile(comm, filename, interpolate, dm));
5259   } else if (isPLY) {
5260     PetscCall(DMPlexCreatePLYFromFile(comm, filename, interpolate, dm));
5261   } else if (isEGADSLite || isEGADS || isIGES || isSTEP) {
5262     if (isEGADSLite) PetscCall(DMPlexCreateEGADSLiteFromFile(comm, filename, dm));
5263     else PetscCall(DMPlexCreateEGADSFromFile(comm, filename, dm));
5264     if (!interpolate) {
5265       DM udm;
5266 
5267       PetscCall(DMPlexUninterpolate(*dm, &udm));
5268       PetscCall(DMDestroy(dm));
5269       *dm = udm;
5270     }
5271   } else if (isCV) {
5272     PetscCall(DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm));
5273   } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
5274   PetscCall(PetscStrlen(plexname, &len));
5275   if (len) PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5276   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5277   PetscFunctionReturn(0);
5278 }
5279 /*@C
5280   DMPlexCreateEphemeral - This takes a `DMPlexTransform` and a base `DMPlex` and produces an ephemeral `DM`, meaning one that is created on the fly in response to queries.
5281 
5282   Input Parameter:
5283 . tr - The `DMPlexTransform`
5284 
5285   Output Parameter:
5286 . dm - The `DM`
5287 
5288   Notes:
5289   An emphemeral mesh is one that is not stored concretely, as in the default Plex implementation, but rather is produced on the fly in response to queries, using information from the transform and the base mesh.
5290 
5291   Level: beginner
5292 
5293 .seealso: `DMPlexCreateFromFile`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
5294 @*/
5295 PetscErrorCode DMPlexCreateEphemeral(DMPlexTransform tr, DM *dm)
5296 {
5297   DM       bdm;
5298   PetscInt Nl;
5299 
5300   PetscFunctionBegin;
5301   PetscCall(DMCreate(PetscObjectComm((PetscObject)tr), dm));
5302   PetscCall(DMSetType(*dm, DMPLEX));
5303   PetscCall(DMSetFromOptions(*dm));
5304 
5305   PetscCall(PetscObjectReference((PetscObject)tr));
5306   PetscCall(DMPlexTransformDestroy(&((DM_Plex *)(*dm)->data)->tr));
5307   ((DM_Plex *)(*dm)->data)->tr = tr;
5308 
5309   PetscCall(DMPlexTransformGetDM(tr, &bdm));
5310   PetscCall(DMGetNumLabels(bdm, &Nl));
5311   for (PetscInt l = 0; l < Nl; ++l) {
5312     DMLabel     label, labelNew;
5313     const char *lname;
5314     PetscBool   isDepth, isCellType;
5315 
5316     PetscCall(DMGetLabelName(bdm, l, &lname));
5317     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
5318     if (isDepth) continue;
5319     PetscCall(PetscStrcmp(lname, "celltype", &isCellType));
5320     if (isCellType) continue;
5321     PetscCall(DMCreateLabel(*dm, lname));
5322     PetscCall(DMGetLabel(bdm, lname, &label));
5323     PetscCall(DMGetLabel(*dm, lname, &labelNew));
5324     PetscCall(DMLabelSetType(labelNew, DMLABELEPHEMERAL));
5325     PetscCall(DMLabelEphemeralSetLabel(labelNew, label));
5326     PetscCall(DMLabelEphemeralSetTransform(labelNew, tr));
5327     PetscCall(DMLabelSetUp(labelNew));
5328   }
5329   PetscFunctionReturn(0);
5330 }
5331