xref: /petsc/src/dm/impls/plex/plexcreate.c (revision 5515ebd38caf9dfb24758a1fa7a0ef84b8937157)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h> /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>   /*I   "petscdmplex.h"   I*/
4 #include <petscsf.h>
5 #include <petscdmplextransform.h>
6 #include <petscdmlabelephemeral.h>
7 #include <petsc/private/kernels/blockmatmult.h>
8 #include <petsc/private/kernels/blockinvert.h>
9 
10 PetscLogEvent DMPLEX_CreateFromFile, DMPLEX_BuildFromCellList, DMPLEX_BuildCoordinatesFromCellList;
11 
12 /* External function declarations here */
13 static PetscErrorCode DMInitialize_Plex(DM dm);
14 
15 /* This copies internal things in the Plex structure that we generally want when making a new, related Plex */
16 PetscErrorCode DMPlexCopy_Internal(DM dmin, PetscBool copyPeriodicity, PetscBool copyOverlap, DM dmout)
17 {
18   const PetscReal         *maxCell, *Lstart, *L;
19   PetscBool                dist;
20   DMPlexReorderDefaultFlag reorder;
21 
22   PetscFunctionBegin;
23   if (copyPeriodicity) {
24     PetscCall(DMGetPeriodicity(dmin, &maxCell, &Lstart, &L));
25     PetscCall(DMSetPeriodicity(dmout, maxCell, Lstart, L));
26   }
27   PetscCall(DMPlexDistributeGetDefault(dmin, &dist));
28   PetscCall(DMPlexDistributeSetDefault(dmout, dist));
29   PetscCall(DMPlexReorderGetDefault(dmin, &reorder));
30   PetscCall(DMPlexReorderSetDefault(dmout, reorder));
31   ((DM_Plex *)dmout->data)->useHashLocation = ((DM_Plex *)dmin->data)->useHashLocation;
32   if (copyOverlap) PetscCall(DMPlexSetOverlap_Plex(dmout, dmin, 0));
33   PetscFunctionReturn(0);
34 }
35 
36 /* Replace dm with the contents of ndm, and then destroy ndm
37    - Share the DM_Plex structure
38    - Share the coordinates
39    - Share the SF
40 */
41 PetscErrorCode DMPlexReplace_Internal(DM dm, DM *ndm)
42 {
43   PetscSF          sf;
44   DM               dmNew = *ndm, coordDM, coarseDM;
45   Vec              coords;
46   const PetscReal *maxCell, *Lstart, *L;
47   PetscInt         dim, cdim;
48 
49   PetscFunctionBegin;
50   if (dm == dmNew) {
51     PetscCall(DMDestroy(ndm));
52     PetscFunctionReturn(0);
53   }
54   dm->setupcalled = dmNew->setupcalled;
55   PetscCall(DMGetDimension(dmNew, &dim));
56   PetscCall(DMSetDimension(dm, dim));
57   PetscCall(DMGetCoordinateDim(dmNew, &cdim));
58   PetscCall(DMSetCoordinateDim(dm, cdim));
59   PetscCall(DMGetPointSF(dmNew, &sf));
60   PetscCall(DMSetPointSF(dm, sf));
61   PetscCall(DMGetCoordinateDM(dmNew, &coordDM));
62   PetscCall(DMGetCoordinatesLocal(dmNew, &coords));
63   PetscCall(DMSetCoordinateDM(dm, coordDM));
64   PetscCall(DMSetCoordinatesLocal(dm, coords));
65   PetscCall(DMGetCellCoordinateDM(dmNew, &coordDM));
66   PetscCall(DMGetCellCoordinatesLocal(dmNew, &coords));
67   PetscCall(DMSetCellCoordinateDM(dm, coordDM));
68   PetscCall(DMSetCellCoordinatesLocal(dm, coords));
69   /* Do not want to create the coordinate field if it does not already exist, so do not call DMGetCoordinateField() */
70   PetscCall(DMFieldDestroy(&dm->coordinates[0].field));
71   dm->coordinates[0].field            = dmNew->coordinates[0].field;
72   ((DM_Plex *)dmNew->data)->coordFunc = ((DM_Plex *)dm->data)->coordFunc;
73   PetscCall(DMGetPeriodicity(dmNew, &maxCell, &Lstart, &L));
74   PetscCall(DMSetPeriodicity(dm, maxCell, Lstart, L));
75   PetscCall(DMDestroy_Plex(dm));
76   PetscCall(DMInitialize_Plex(dm));
77   dm->data = dmNew->data;
78   ((DM_Plex *)dmNew->data)->refct++;
79   PetscCall(DMDestroyLabelLinkList_Internal(dm));
80   PetscCall(DMCopyLabels(dmNew, dm, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL));
81   PetscCall(DMGetCoarseDM(dmNew, &coarseDM));
82   PetscCall(DMSetCoarseDM(dm, coarseDM));
83   PetscCall(DMDestroy(ndm));
84   PetscFunctionReturn(0);
85 }
86 
87 /* Swap dm with the contents of dmNew
88    - Swap the DM_Plex structure
89    - Swap the coordinates
90    - Swap the point PetscSF
91 */
92 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
93 {
94   DM          coordDMA, coordDMB;
95   Vec         coordsA, coordsB;
96   PetscSF     sfA, sfB;
97   DMField     fieldTmp;
98   void       *tmp;
99   DMLabelLink listTmp;
100   DMLabel     depthTmp;
101   PetscInt    tmpI;
102 
103   PetscFunctionBegin;
104   if (dmA == dmB) PetscFunctionReturn(0);
105   PetscCall(DMGetPointSF(dmA, &sfA));
106   PetscCall(DMGetPointSF(dmB, &sfB));
107   PetscCall(PetscObjectReference((PetscObject)sfA));
108   PetscCall(DMSetPointSF(dmA, sfB));
109   PetscCall(DMSetPointSF(dmB, sfA));
110   PetscCall(PetscObjectDereference((PetscObject)sfA));
111 
112   PetscCall(DMGetCoordinateDM(dmA, &coordDMA));
113   PetscCall(DMGetCoordinateDM(dmB, &coordDMB));
114   PetscCall(PetscObjectReference((PetscObject)coordDMA));
115   PetscCall(DMSetCoordinateDM(dmA, coordDMB));
116   PetscCall(DMSetCoordinateDM(dmB, coordDMA));
117   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
118 
119   PetscCall(DMGetCoordinatesLocal(dmA, &coordsA));
120   PetscCall(DMGetCoordinatesLocal(dmB, &coordsB));
121   PetscCall(PetscObjectReference((PetscObject)coordsA));
122   PetscCall(DMSetCoordinatesLocal(dmA, coordsB));
123   PetscCall(DMSetCoordinatesLocal(dmB, coordsA));
124   PetscCall(PetscObjectDereference((PetscObject)coordsA));
125 
126   PetscCall(DMGetCellCoordinateDM(dmA, &coordDMA));
127   PetscCall(DMGetCellCoordinateDM(dmB, &coordDMB));
128   PetscCall(PetscObjectReference((PetscObject)coordDMA));
129   PetscCall(DMSetCellCoordinateDM(dmA, coordDMB));
130   PetscCall(DMSetCellCoordinateDM(dmB, coordDMA));
131   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
132 
133   PetscCall(DMGetCellCoordinatesLocal(dmA, &coordsA));
134   PetscCall(DMGetCellCoordinatesLocal(dmB, &coordsB));
135   PetscCall(PetscObjectReference((PetscObject)coordsA));
136   PetscCall(DMSetCellCoordinatesLocal(dmA, coordsB));
137   PetscCall(DMSetCellCoordinatesLocal(dmB, coordsA));
138   PetscCall(PetscObjectDereference((PetscObject)coordsA));
139 
140   fieldTmp                  = dmA->coordinates[0].field;
141   dmA->coordinates[0].field = dmB->coordinates[0].field;
142   dmB->coordinates[0].field = fieldTmp;
143   fieldTmp                  = dmA->coordinates[1].field;
144   dmA->coordinates[1].field = dmB->coordinates[1].field;
145   dmB->coordinates[1].field = fieldTmp;
146   tmp                       = dmA->data;
147   dmA->data                 = dmB->data;
148   dmB->data                 = tmp;
149   listTmp                   = dmA->labels;
150   dmA->labels               = dmB->labels;
151   dmB->labels               = listTmp;
152   depthTmp                  = dmA->depthLabel;
153   dmA->depthLabel           = dmB->depthLabel;
154   dmB->depthLabel           = depthTmp;
155   depthTmp                  = dmA->celltypeLabel;
156   dmA->celltypeLabel        = dmB->celltypeLabel;
157   dmB->celltypeLabel        = depthTmp;
158   tmpI                      = dmA->levelup;
159   dmA->levelup              = dmB->levelup;
160   dmB->levelup              = tmpI;
161   PetscFunctionReturn(0);
162 }
163 
164 static PetscErrorCode DMPlexInterpolateInPlace_Internal(DM dm)
165 {
166   DM idm;
167 
168   PetscFunctionBegin;
169   PetscCall(DMPlexInterpolate(dm, &idm));
170   PetscCall(DMPlexCopyCoordinates(dm, idm));
171   PetscCall(DMPlexReplace_Internal(dm, &idm));
172   PetscFunctionReturn(0);
173 }
174 
175 /*@C
176   DMPlexCreateCoordinateSpace - Creates a finite element space for the coordinates
177 
178   Collective on dm
179 
180   Input Parameters:
181 + DM        - The DM
182 . degree    - The degree of the finite element or PETSC_DECIDE
183 - coordFunc - An optional function to map new points from refinement to the surface
184 
185   Level: advanced
186 
187 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `PetscPointFunc`, `PetscFECreateLagrange()`, `DMGetCoordinateDM()`
188 @*/
189 PetscErrorCode DMPlexCreateCoordinateSpace(DM dm, PetscInt degree, PetscPointFunc coordFunc)
190 {
191   DM_Plex     *mesh = (DM_Plex *)dm->data;
192   DM           cdm;
193   PetscDS      cds;
194   PetscFE      fe;
195   PetscClassId id;
196 
197   PetscFunctionBegin;
198   PetscCall(DMGetCoordinateDM(dm, &cdm));
199   PetscCall(DMGetDS(cdm, &cds));
200   PetscCall(PetscDSGetDiscretization(cds, 0, (PetscObject *)&fe));
201   PetscCall(PetscObjectGetClassId((PetscObject)fe, &id));
202   if (id != PETSCFE_CLASSID) {
203     PetscInt dim, dE, qorder;
204 
205     PetscCall(DMGetDimension(dm, &dim));
206     PetscCall(DMGetCoordinateDim(dm, &dE));
207     qorder = degree;
208     PetscObjectOptionsBegin((PetscObject)cdm);
209     PetscCall(PetscOptionsBoundedInt("-default_quadrature_order", "Quadrature order is one less than quadrature points per edge", "DMPlexCreateCoordinateSpace", qorder, &qorder, NULL, 0));
210     PetscOptionsEnd();
211     if (degree == PETSC_DECIDE) fe = NULL;
212     else {
213       DMPolytopeType ct;
214       PetscInt       cStart, cEnd, ctTmp;
215 
216       PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
217       // Want to match cell types
218       if (cEnd > cStart) PetscCall(DMPlexGetCellType(dm, cStart, &ct));
219       else ct = DM_POLYTOPE_UNKNOWN;
220       ctTmp = (PetscInt)ct;
221       PetscCallMPI(MPI_Allreduce(MPI_IN_PLACE, &ctTmp, 1, MPIU_INT, MPI_MIN, PetscObjectComm((PetscObject)dm)));
222       ct = (DMPolytopeType)ctTmp;
223       // Work around current bug
224       if (ct == DM_POLYTOPE_SEG_PRISM_TENSOR || ct == DM_POLYTOPE_TRI_PRISM_TENSOR || ct == DM_POLYTOPE_QUAD_PRISM_TENSOR) fe = NULL;
225       else PetscCall(PetscFECreateLagrangeByCell(PETSC_COMM_SELF, dim, dE, ct, degree, qorder, &fe));
226     }
227     if (fe) PetscCall(DMProjectCoordinates(dm, fe));
228     PetscCall(PetscFEDestroy(&fe));
229   }
230   mesh->coordFunc = coordFunc;
231   PetscFunctionReturn(0);
232 }
233 
234 /*@
235   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
236 
237   Collective
238 
239   Input Parameters:
240 + comm - The communicator for the `DM` object
241 . dim - The spatial dimension
242 . simplex - Flag for simplicial cells, otherwise they are tensor product cells
243 . interpolate - Flag to create intermediate mesh pieces (edges, faces)
244 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
245 
246   Output Parameter:
247 . dm - The `DM` object
248 
249   Level: beginner
250 
251 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMSetType()`, `DMCreate()`
252 @*/
253 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscReal refinementLimit, DM *newdm)
254 {
255   DM          dm;
256   PetscMPIInt rank;
257 
258   PetscFunctionBegin;
259   PetscCall(DMCreate(comm, &dm));
260   PetscCall(DMSetType(dm, DMPLEX));
261   PetscCall(DMSetDimension(dm, dim));
262   PetscCallMPI(MPI_Comm_rank(comm, &rank));
263   switch (dim) {
264   case 2:
265     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "triangular"));
266     else PetscCall(PetscObjectSetName((PetscObject)dm, "quadrilateral"));
267     break;
268   case 3:
269     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "tetrahedral"));
270     else PetscCall(PetscObjectSetName((PetscObject)dm, "hexahedral"));
271     break;
272   default:
273     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
274   }
275   if (rank) {
276     PetscInt numPoints[2] = {0, 0};
277     PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL));
278   } else {
279     switch (dim) {
280     case 2:
281       if (simplex) {
282         PetscInt    numPoints[2]        = {4, 2};
283         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
284         PetscInt    cones[6]            = {2, 3, 4, 5, 4, 3};
285         PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
286         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
287 
288         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
289       } else {
290         PetscInt    numPoints[2]        = {6, 2};
291         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
292         PetscInt    cones[8]            = {2, 3, 4, 5, 3, 6, 7, 4};
293         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
294         PetscScalar vertexCoords[12]    = {-1.0, -0.5, 0.0, -0.5, 0.0, 0.5, -1.0, 0.5, 1.0, -0.5, 1.0, 0.5};
295 
296         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
297       }
298       break;
299     case 3:
300       if (simplex) {
301         PetscInt    numPoints[2]        = {5, 2};
302         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
303         PetscInt    cones[8]            = {4, 3, 5, 2, 5, 3, 4, 6};
304         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
305         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0};
306 
307         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
308       } else {
309         PetscInt    numPoints[2]         = {12, 2};
310         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
311         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9, 5, 4, 10, 11, 7, 12, 13, 8};
312         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
313         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5, -1.0, 0.5, -0.5, 0.0, 0.5, -0.5, 0.0, -0.5, -0.5, -1.0, -0.5, 0.5, 0.0, -0.5, 0.5, 0.0, 0.5, 0.5, -1.0, 0.5, 0.5, 1.0, 0.5, -0.5, 1.0, -0.5, -0.5, 1.0, -0.5, 0.5, 1.0, 0.5, 0.5};
314 
315         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
316       }
317       break;
318     default:
319       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
320     }
321   }
322   *newdm = dm;
323   if (refinementLimit > 0.0) {
324     DM          rdm;
325     const char *name;
326 
327     PetscCall(DMPlexSetRefinementUniform(*newdm, PETSC_FALSE));
328     PetscCall(DMPlexSetRefinementLimit(*newdm, refinementLimit));
329     PetscCall(DMRefine(*newdm, comm, &rdm));
330     PetscCall(PetscObjectGetName((PetscObject)*newdm, &name));
331     PetscCall(PetscObjectSetName((PetscObject)rdm, name));
332     PetscCall(DMDestroy(newdm));
333     *newdm = rdm;
334   }
335   if (interpolate) {
336     DM idm;
337 
338     PetscCall(DMPlexInterpolate(*newdm, &idm));
339     PetscCall(DMDestroy(newdm));
340     *newdm = idm;
341   }
342   PetscFunctionReturn(0);
343 }
344 
345 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
346 {
347   const PetscInt numVertices    = 2;
348   PetscInt       markerRight    = 1;
349   PetscInt       markerLeft     = 1;
350   PetscBool      markerSeparate = PETSC_FALSE;
351   Vec            coordinates;
352   PetscSection   coordSection;
353   PetscScalar   *coords;
354   PetscInt       coordSize;
355   PetscMPIInt    rank;
356   PetscInt       cdim = 1, v;
357 
358   PetscFunctionBegin;
359   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
360   if (markerSeparate) {
361     markerRight = 2;
362     markerLeft  = 1;
363   }
364   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
365   if (rank == 0) {
366     PetscCall(DMPlexSetChart(dm, 0, numVertices));
367     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
368     PetscCall(DMSetLabelValue(dm, "marker", 0, markerLeft));
369     PetscCall(DMSetLabelValue(dm, "marker", 1, markerRight));
370   }
371   PetscCall(DMPlexSymmetrize(dm));
372   PetscCall(DMPlexStratify(dm));
373   /* Build coordinates */
374   PetscCall(DMSetCoordinateDim(dm, cdim));
375   PetscCall(DMGetCoordinateSection(dm, &coordSection));
376   PetscCall(PetscSectionSetNumFields(coordSection, 1));
377   PetscCall(PetscSectionSetChart(coordSection, 0, numVertices));
378   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
379   for (v = 0; v < numVertices; ++v) {
380     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
381     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
382   }
383   PetscCall(PetscSectionSetUp(coordSection));
384   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
385   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
386   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
387   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
388   PetscCall(VecSetBlockSize(coordinates, cdim));
389   PetscCall(VecSetType(coordinates, VECSTANDARD));
390   PetscCall(VecGetArray(coordinates, &coords));
391   coords[0] = lower[0];
392   coords[1] = upper[0];
393   PetscCall(VecRestoreArray(coordinates, &coords));
394   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
395   PetscCall(VecDestroy(&coordinates));
396   PetscFunctionReturn(0);
397 }
398 
399 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
400 {
401   const PetscInt numVertices    = (edges[0] + 1) * (edges[1] + 1);
402   const PetscInt numEdges       = edges[0] * (edges[1] + 1) + (edges[0] + 1) * edges[1];
403   PetscInt       markerTop      = 1;
404   PetscInt       markerBottom   = 1;
405   PetscInt       markerRight    = 1;
406   PetscInt       markerLeft     = 1;
407   PetscBool      markerSeparate = PETSC_FALSE;
408   Vec            coordinates;
409   PetscSection   coordSection;
410   PetscScalar   *coords;
411   PetscInt       coordSize;
412   PetscMPIInt    rank;
413   PetscInt       v, vx, vy;
414 
415   PetscFunctionBegin;
416   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
417   if (markerSeparate) {
418     markerTop    = 3;
419     markerBottom = 1;
420     markerRight  = 2;
421     markerLeft   = 4;
422   }
423   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
424   if (rank == 0) {
425     PetscInt e, ex, ey;
426 
427     PetscCall(DMPlexSetChart(dm, 0, numEdges + numVertices));
428     for (e = 0; e < numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
429     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
430     for (vx = 0; vx <= edges[0]; vx++) {
431       for (ey = 0; ey < edges[1]; ey++) {
432         PetscInt edge   = vx * edges[1] + ey + edges[0] * (edges[1] + 1);
433         PetscInt vertex = ey * (edges[0] + 1) + vx + numEdges;
434         PetscInt cone[2];
435 
436         cone[0] = vertex;
437         cone[1] = vertex + edges[0] + 1;
438         PetscCall(DMPlexSetCone(dm, edge, cone));
439         if (vx == edges[0]) {
440           PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
441           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
442           if (ey == edges[1] - 1) {
443             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
444             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerRight));
445           }
446         } else if (vx == 0) {
447           PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
448           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
449           if (ey == edges[1] - 1) {
450             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
451             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft));
452           }
453         }
454       }
455     }
456     for (vy = 0; vy <= edges[1]; vy++) {
457       for (ex = 0; ex < edges[0]; ex++) {
458         PetscInt edge   = vy * edges[0] + ex;
459         PetscInt vertex = vy * (edges[0] + 1) + ex + numEdges;
460         PetscInt cone[2];
461 
462         cone[0] = vertex;
463         cone[1] = vertex + 1;
464         PetscCall(DMPlexSetCone(dm, edge, cone));
465         if (vy == edges[1]) {
466           PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
467           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
468           if (ex == edges[0] - 1) {
469             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
470             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerTop));
471           }
472         } else if (vy == 0) {
473           PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
474           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
475           if (ex == edges[0] - 1) {
476             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
477             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom));
478           }
479         }
480       }
481     }
482   }
483   PetscCall(DMPlexSymmetrize(dm));
484   PetscCall(DMPlexStratify(dm));
485   /* Build coordinates */
486   PetscCall(DMSetCoordinateDim(dm, 2));
487   PetscCall(DMGetCoordinateSection(dm, &coordSection));
488   PetscCall(PetscSectionSetNumFields(coordSection, 1));
489   PetscCall(PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices));
490   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 2));
491   for (v = numEdges; v < numEdges + numVertices; ++v) {
492     PetscCall(PetscSectionSetDof(coordSection, v, 2));
493     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 2));
494   }
495   PetscCall(PetscSectionSetUp(coordSection));
496   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
497   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
498   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
499   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
500   PetscCall(VecSetBlockSize(coordinates, 2));
501   PetscCall(VecSetType(coordinates, VECSTANDARD));
502   PetscCall(VecGetArray(coordinates, &coords));
503   for (vy = 0; vy <= edges[1]; ++vy) {
504     for (vx = 0; vx <= edges[0]; ++vx) {
505       coords[(vy * (edges[0] + 1) + vx) * 2 + 0] = lower[0] + ((upper[0] - lower[0]) / edges[0]) * vx;
506       coords[(vy * (edges[0] + 1) + vx) * 2 + 1] = lower[1] + ((upper[1] - lower[1]) / edges[1]) * vy;
507     }
508   }
509   PetscCall(VecRestoreArray(coordinates, &coords));
510   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
511   PetscCall(VecDestroy(&coordinates));
512   PetscFunctionReturn(0);
513 }
514 
515 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
516 {
517   PetscInt     vertices[3], numVertices;
518   PetscInt     numFaces       = 2 * faces[0] * faces[1] + 2 * faces[1] * faces[2] + 2 * faces[0] * faces[2];
519   PetscInt     markerTop      = 1;
520   PetscInt     markerBottom   = 1;
521   PetscInt     markerFront    = 1;
522   PetscInt     markerBack     = 1;
523   PetscInt     markerRight    = 1;
524   PetscInt     markerLeft     = 1;
525   PetscBool    markerSeparate = PETSC_FALSE;
526   Vec          coordinates;
527   PetscSection coordSection;
528   PetscScalar *coords;
529   PetscInt     coordSize;
530   PetscMPIInt  rank;
531   PetscInt     v, vx, vy, vz;
532   PetscInt     voffset, iface = 0, cone[4];
533 
534   PetscFunctionBegin;
535   PetscCheck(faces[0] >= 1 && faces[1] >= 1 && faces[2] >= 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
536   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
537   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
538   if (markerSeparate) {
539     markerBottom = 1;
540     markerTop    = 2;
541     markerFront  = 3;
542     markerBack   = 4;
543     markerRight  = 5;
544     markerLeft   = 6;
545   }
546   vertices[0] = faces[0] + 1;
547   vertices[1] = faces[1] + 1;
548   vertices[2] = faces[2] + 1;
549   numVertices = vertices[0] * vertices[1] * vertices[2];
550   if (rank == 0) {
551     PetscInt f;
552 
553     PetscCall(DMPlexSetChart(dm, 0, numFaces + numVertices));
554     for (f = 0; f < numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
555     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
556 
557     /* Side 0 (Top) */
558     for (vy = 0; vy < faces[1]; vy++) {
559       for (vx = 0; vx < faces[0]; vx++) {
560         voffset = numFaces + vertices[0] * vertices[1] * (vertices[2] - 1) + vy * vertices[0] + vx;
561         cone[0] = voffset;
562         cone[1] = voffset + 1;
563         cone[2] = voffset + vertices[0] + 1;
564         cone[3] = voffset + vertices[0];
565         PetscCall(DMPlexSetCone(dm, iface, cone));
566         PetscCall(DMSetLabelValue(dm, "marker", iface, markerTop));
567         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerTop));
568         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerTop));
569         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerTop));
570         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerTop));
571         iface++;
572       }
573     }
574 
575     /* Side 1 (Bottom) */
576     for (vy = 0; vy < faces[1]; vy++) {
577       for (vx = 0; vx < faces[0]; vx++) {
578         voffset = numFaces + vy * (faces[0] + 1) + vx;
579         cone[0] = voffset + 1;
580         cone[1] = voffset;
581         cone[2] = voffset + vertices[0];
582         cone[3] = voffset + vertices[0] + 1;
583         PetscCall(DMPlexSetCone(dm, iface, cone));
584         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBottom));
585         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBottom));
586         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBottom));
587         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerBottom));
588         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerBottom));
589         iface++;
590       }
591     }
592 
593     /* Side 2 (Front) */
594     for (vz = 0; vz < faces[2]; vz++) {
595       for (vx = 0; vx < faces[0]; vx++) {
596         voffset = numFaces + vz * vertices[0] * vertices[1] + vx;
597         cone[0] = voffset;
598         cone[1] = voffset + 1;
599         cone[2] = voffset + vertices[0] * vertices[1] + 1;
600         cone[3] = voffset + vertices[0] * vertices[1];
601         PetscCall(DMPlexSetCone(dm, iface, cone));
602         PetscCall(DMSetLabelValue(dm, "marker", iface, markerFront));
603         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerFront));
604         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerFront));
605         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerFront));
606         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerFront));
607         iface++;
608       }
609     }
610 
611     /* Side 3 (Back) */
612     for (vz = 0; vz < faces[2]; vz++) {
613       for (vx = 0; vx < faces[0]; vx++) {
614         voffset = numFaces + vz * vertices[0] * vertices[1] + vertices[0] * (vertices[1] - 1) + vx;
615         cone[0] = voffset + vertices[0] * vertices[1];
616         cone[1] = voffset + vertices[0] * vertices[1] + 1;
617         cone[2] = voffset + 1;
618         cone[3] = voffset;
619         PetscCall(DMPlexSetCone(dm, iface, cone));
620         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBack));
621         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBack));
622         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBack));
623         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerBack));
624         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerBack));
625         iface++;
626       }
627     }
628 
629     /* Side 4 (Left) */
630     for (vz = 0; vz < faces[2]; vz++) {
631       for (vy = 0; vy < faces[1]; vy++) {
632         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0];
633         cone[0] = voffset;
634         cone[1] = voffset + vertices[0] * vertices[1];
635         cone[2] = voffset + vertices[0] * vertices[1] + vertices[0];
636         cone[3] = voffset + vertices[0];
637         PetscCall(DMPlexSetCone(dm, iface, cone));
638         PetscCall(DMSetLabelValue(dm, "marker", iface, markerLeft));
639         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerLeft));
640         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerLeft));
641         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[1] + 0, markerLeft));
642         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerLeft));
643         iface++;
644       }
645     }
646 
647     /* Side 5 (Right) */
648     for (vz = 0; vz < faces[2]; vz++) {
649       for (vy = 0; vy < faces[1]; vy++) {
650         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0] + faces[0];
651         cone[0] = voffset + vertices[0] * vertices[1];
652         cone[1] = voffset;
653         cone[2] = voffset + vertices[0];
654         cone[3] = voffset + vertices[0] * vertices[1] + vertices[0];
655         PetscCall(DMPlexSetCone(dm, iface, cone));
656         PetscCall(DMSetLabelValue(dm, "marker", iface, markerRight));
657         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerRight));
658         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerRight));
659         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerRight));
660         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerRight));
661         iface++;
662       }
663     }
664   }
665   PetscCall(DMPlexSymmetrize(dm));
666   PetscCall(DMPlexStratify(dm));
667   /* Build coordinates */
668   PetscCall(DMSetCoordinateDim(dm, 3));
669   PetscCall(DMGetCoordinateSection(dm, &coordSection));
670   PetscCall(PetscSectionSetNumFields(coordSection, 1));
671   PetscCall(PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices));
672   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 3));
673   for (v = numFaces; v < numFaces + numVertices; ++v) {
674     PetscCall(PetscSectionSetDof(coordSection, v, 3));
675     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 3));
676   }
677   PetscCall(PetscSectionSetUp(coordSection));
678   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
679   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
680   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
681   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
682   PetscCall(VecSetBlockSize(coordinates, 3));
683   PetscCall(VecSetType(coordinates, VECSTANDARD));
684   PetscCall(VecGetArray(coordinates, &coords));
685   for (vz = 0; vz <= faces[2]; ++vz) {
686     for (vy = 0; vy <= faces[1]; ++vy) {
687       for (vx = 0; vx <= faces[0]; ++vx) {
688         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 0] = lower[0] + ((upper[0] - lower[0]) / faces[0]) * vx;
689         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 1] = lower[1] + ((upper[1] - lower[1]) / faces[1]) * vy;
690         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 2] = lower[2] + ((upper[2] - lower[2]) / faces[2]) * vz;
691       }
692     }
693   }
694   PetscCall(VecRestoreArray(coordinates, &coords));
695   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
696   PetscCall(VecDestroy(&coordinates));
697   PetscFunctionReturn(0);
698 }
699 
700 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate)
701 {
702   PetscFunctionBegin;
703   PetscValidLogicalCollectiveInt(dm, dim, 2);
704   PetscCall(DMSetDimension(dm, dim - 1));
705   PetscCall(DMSetCoordinateDim(dm, dim));
706   switch (dim) {
707   case 1:
708     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(dm, lower, upper, faces));
709     break;
710   case 2:
711     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(dm, lower, upper, faces));
712     break;
713   case 3:
714     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(dm, lower, upper, faces));
715     break;
716   default:
717     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Dimension not supported: %" PetscInt_FMT, dim);
718   }
719   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
720   PetscFunctionReturn(0);
721 }
722 
723 /*@C
724   DMPlexCreateBoxSurfaceMesh - Creates a mesh on the surface of the tensor product of unit intervals (box) using tensor cells (hexahedra).
725 
726   Collective
727 
728   Input Parameters:
729 + comm        - The communicator for the `DM` object
730 . dim         - The spatial dimension of the box, so the resulting mesh is has dimension dim-1
731 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
732 . lower       - The lower left corner, or NULL for (0, 0, 0)
733 . upper       - The upper right corner, or NULL for (1, 1, 1)
734 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
735 
736   Output Parameter:
737 . dm  - The `DM` object
738 
739   Level: beginner
740 
741 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateBoxMesh()`, `DMPlexCreateFromFile()`, `DMSetType()`, `DMCreate()`
742 @*/
743 PetscErrorCode DMPlexCreateBoxSurfaceMesh(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate, DM *dm)
744 {
745   PetscInt  fac[3] = {1, 1, 1};
746   PetscReal low[3] = {0, 0, 0};
747   PetscReal upp[3] = {1, 1, 1};
748 
749   PetscFunctionBegin;
750   PetscCall(DMCreate(comm, dm));
751   PetscCall(DMSetType(*dm, DMPLEX));
752   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(*dm, dim, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, interpolate));
753   PetscFunctionReturn(0);
754 }
755 
756 static PetscErrorCode DMPlexCreateLineMesh_Internal(DM dm, PetscInt segments, PetscReal lower, PetscReal upper, DMBoundaryType bd)
757 {
758   PetscInt     i, fStart, fEnd, numCells = 0, numVerts = 0;
759   PetscInt     numPoints[2], *coneSize, *cones, *coneOrientations;
760   PetscScalar *vertexCoords;
761   PetscReal    L, maxCell;
762   PetscBool    markerSeparate = PETSC_FALSE;
763   PetscInt     markerLeft = 1, faceMarkerLeft = 1;
764   PetscInt     markerRight = 1, faceMarkerRight = 2;
765   PetscBool    wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
766   PetscMPIInt  rank;
767 
768   PetscFunctionBegin;
769   PetscValidPointer(dm, 1);
770 
771   PetscCall(DMSetDimension(dm, 1));
772   PetscCall(DMCreateLabel(dm, "marker"));
773   PetscCall(DMCreateLabel(dm, "Face Sets"));
774 
775   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
776   if (rank == 0) numCells = segments;
777   if (rank == 0) numVerts = segments + (wrap ? 0 : 1);
778 
779   numPoints[0] = numVerts;
780   numPoints[1] = numCells;
781   PetscCall(PetscMalloc4(numCells + numVerts, &coneSize, numCells * 2, &cones, numCells + numVerts, &coneOrientations, numVerts, &vertexCoords));
782   PetscCall(PetscArrayzero(coneOrientations, numCells + numVerts));
783   for (i = 0; i < numCells; ++i) coneSize[i] = 2;
784   for (i = 0; i < numVerts; ++i) coneSize[numCells + i] = 0;
785   for (i = 0; i < numCells; ++i) {
786     cones[2 * i]     = numCells + i % numVerts;
787     cones[2 * i + 1] = numCells + (i + 1) % numVerts;
788   }
789   for (i = 0; i < numVerts; ++i) vertexCoords[i] = lower + (upper - lower) * ((PetscReal)i / (PetscReal)numCells);
790   PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
791   PetscCall(PetscFree4(coneSize, cones, coneOrientations, vertexCoords));
792 
793   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
794   if (markerSeparate) {
795     markerLeft  = faceMarkerLeft;
796     markerRight = faceMarkerRight;
797   }
798   if (!wrap && rank == 0) {
799     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
800     PetscCall(DMSetLabelValue(dm, "marker", fStart, markerLeft));
801     PetscCall(DMSetLabelValue(dm, "marker", fEnd - 1, markerRight));
802     PetscCall(DMSetLabelValue(dm, "Face Sets", fStart, faceMarkerLeft));
803     PetscCall(DMSetLabelValue(dm, "Face Sets", fEnd - 1, faceMarkerRight));
804   }
805   if (wrap) {
806     L       = upper - lower;
807     maxCell = (PetscReal)1.1 * (L / (PetscReal)PetscMax(1, segments));
808     PetscCall(DMSetPeriodicity(dm, &maxCell, &lower, &L));
809   }
810   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
811   PetscFunctionReturn(0);
812 }
813 
814 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
815 {
816   DM      boundary, vol;
817   DMLabel bdlabel;
818 
819   PetscFunctionBegin;
820   PetscValidPointer(dm, 1);
821   for (PetscInt i = 0; i < dim; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
822   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &boundary));
823   PetscCall(DMSetType(boundary, DMPLEX));
824   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(boundary, dim, faces, lower, upper, PETSC_FALSE));
825   PetscCall(DMPlexGenerate(boundary, NULL, interpolate, &vol));
826   PetscCall(DMGetLabel(vol, "marker", &bdlabel));
827   if (bdlabel) PetscCall(DMPlexLabelComplete(vol, bdlabel));
828   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_FALSE, vol));
829   PetscCall(DMPlexReplace_Internal(dm, &vol));
830   PetscCall(DMDestroy(&boundary));
831   PetscFunctionReturn(0);
832 }
833 
834 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
835 {
836   DMLabel     cutLabel  = NULL;
837   PetscInt    markerTop = 1, faceMarkerTop = 1;
838   PetscInt    markerBottom = 1, faceMarkerBottom = 1;
839   PetscInt    markerFront = 1, faceMarkerFront = 1;
840   PetscInt    markerBack = 1, faceMarkerBack = 1;
841   PetscInt    markerRight = 1, faceMarkerRight = 1;
842   PetscInt    markerLeft = 1, faceMarkerLeft = 1;
843   PetscInt    dim;
844   PetscBool   markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
845   PetscMPIInt rank;
846 
847   PetscFunctionBegin;
848   PetscCall(DMGetDimension(dm, &dim));
849   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
850   PetscCall(DMCreateLabel(dm, "marker"));
851   PetscCall(DMCreateLabel(dm, "Face Sets"));
852   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
853   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST || bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST || bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
854     if (cutMarker) {
855       PetscCall(DMCreateLabel(dm, "periodic_cut"));
856       PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
857     }
858   }
859   switch (dim) {
860   case 2:
861     faceMarkerTop    = 3;
862     faceMarkerBottom = 1;
863     faceMarkerRight  = 2;
864     faceMarkerLeft   = 4;
865     break;
866   case 3:
867     faceMarkerBottom = 1;
868     faceMarkerTop    = 2;
869     faceMarkerFront  = 3;
870     faceMarkerBack   = 4;
871     faceMarkerRight  = 5;
872     faceMarkerLeft   = 6;
873     break;
874   default:
875     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
876   }
877   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
878   if (markerSeparate) {
879     markerBottom = faceMarkerBottom;
880     markerTop    = faceMarkerTop;
881     markerFront  = faceMarkerFront;
882     markerBack   = faceMarkerBack;
883     markerRight  = faceMarkerRight;
884     markerLeft   = faceMarkerLeft;
885   }
886   {
887     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
888     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
889     const PetscInt numZEdges    = rank == 0 ? edges[2] : 0;
890     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0] + 1) : 0;
891     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1] + 1) : 0;
892     const PetscInt numZVertices = rank == 0 ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2] + 1) : 0;
893     const PetscInt numCells     = numXEdges * numYEdges * numZEdges;
894     const PetscInt numXFaces    = numYEdges * numZEdges;
895     const PetscInt numYFaces    = numXEdges * numZEdges;
896     const PetscInt numZFaces    = numXEdges * numYEdges;
897     const PetscInt numTotXFaces = numXVertices * numXFaces;
898     const PetscInt numTotYFaces = numYVertices * numYFaces;
899     const PetscInt numTotZFaces = numZVertices * numZFaces;
900     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
901     const PetscInt numTotXEdges = numXEdges * numYVertices * numZVertices;
902     const PetscInt numTotYEdges = numYEdges * numXVertices * numZVertices;
903     const PetscInt numTotZEdges = numZEdges * numXVertices * numYVertices;
904     const PetscInt numVertices  = numXVertices * numYVertices * numZVertices;
905     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
906     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
907     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
908     const PetscInt firstYFace   = firstXFace + numTotXFaces;
909     const PetscInt firstZFace   = firstYFace + numTotYFaces;
910     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
911     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
912     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
913     Vec            coordinates;
914     PetscSection   coordSection;
915     PetscScalar   *coords;
916     PetscInt       coordSize;
917     PetscInt       v, vx, vy, vz;
918     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
919 
920     PetscCall(DMPlexSetChart(dm, 0, numCells + numFaces + numEdges + numVertices));
921     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
922     for (f = firstXFace; f < firstXFace + numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
923     for (e = firstXEdge; e < firstXEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
924     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
925     /* Build cells */
926     for (fz = 0; fz < numZEdges; ++fz) {
927       for (fy = 0; fy < numYEdges; ++fy) {
928         for (fx = 0; fx < numXEdges; ++fx) {
929           PetscInt cell  = (fz * numYEdges + fy) * numXEdges + fx;
930           PetscInt faceB = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
931           PetscInt faceT = firstZFace + (fy * numXEdges + fx) * numZVertices + ((fz + 1) % numZVertices);
932           PetscInt faceF = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
933           PetscInt faceK = firstYFace + (fz * numXEdges + fx) * numYVertices + ((fy + 1) % numYVertices);
934           PetscInt faceL = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
935           PetscInt faceR = firstXFace + (fz * numYEdges + fy) * numXVertices + ((fx + 1) % numXVertices);
936           /* B,  T,  F,  K,  R,  L */
937           PetscInt ornt[6] = {-2, 0, 0, -3, 0, -2}; /* ??? */
938           PetscInt cone[6];
939 
940           /* no boundary twisting in 3D */
941           cone[0] = faceB;
942           cone[1] = faceT;
943           cone[2] = faceF;
944           cone[3] = faceK;
945           cone[4] = faceR;
946           cone[5] = faceL;
947           PetscCall(DMPlexSetCone(dm, cell, cone));
948           PetscCall(DMPlexSetConeOrientation(dm, cell, ornt));
949           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
950           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
951           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
952         }
953       }
954     }
955     /* Build x faces */
956     for (fz = 0; fz < numZEdges; ++fz) {
957       for (fy = 0; fy < numYEdges; ++fy) {
958         for (fx = 0; fx < numXVertices; ++fx) {
959           PetscInt face    = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
960           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
961           PetscInt edgeR   = firstZEdge + (((fy + 1) % numYVertices) * numXVertices + fx) * numZEdges + fz;
962           PetscInt edgeB   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
963           PetscInt edgeT   = firstYEdge + (((fz + 1) % numZVertices) * numXVertices + fx) * numYEdges + fy;
964           PetscInt ornt[4] = {0, 0, -1, -1};
965           PetscInt cone[4];
966 
967           if (dim == 3) {
968             /* markers */
969             if (bdX != DM_BOUNDARY_PERIODIC) {
970               if (fx == numXVertices - 1) {
971                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight));
972                 PetscCall(DMSetLabelValue(dm, "marker", face, markerRight));
973               } else if (fx == 0) {
974                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft));
975                 PetscCall(DMSetLabelValue(dm, "marker", face, markerLeft));
976               }
977             }
978           }
979           cone[0] = edgeB;
980           cone[1] = edgeR;
981           cone[2] = edgeT;
982           cone[3] = edgeL;
983           PetscCall(DMPlexSetCone(dm, face, cone));
984           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
985         }
986       }
987     }
988     /* Build y faces */
989     for (fz = 0; fz < numZEdges; ++fz) {
990       for (fx = 0; fx < numXEdges; ++fx) {
991         for (fy = 0; fy < numYVertices; ++fy) {
992           PetscInt face    = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
993           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
994           PetscInt edgeR   = firstZEdge + (fy * numXVertices + ((fx + 1) % numXVertices)) * numZEdges + fz;
995           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
996           PetscInt edgeT   = firstXEdge + (((fz + 1) % numZVertices) * numYVertices + fy) * numXEdges + fx;
997           PetscInt ornt[4] = {0, 0, -1, -1};
998           PetscInt cone[4];
999 
1000           if (dim == 3) {
1001             /* markers */
1002             if (bdY != DM_BOUNDARY_PERIODIC) {
1003               if (fy == numYVertices - 1) {
1004                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack));
1005                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBack));
1006               } else if (fy == 0) {
1007                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront));
1008                 PetscCall(DMSetLabelValue(dm, "marker", face, markerFront));
1009               }
1010             }
1011           }
1012           cone[0] = edgeB;
1013           cone[1] = edgeR;
1014           cone[2] = edgeT;
1015           cone[3] = edgeL;
1016           PetscCall(DMPlexSetCone(dm, face, cone));
1017           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1018         }
1019       }
1020     }
1021     /* Build z faces */
1022     for (fy = 0; fy < numYEdges; ++fy) {
1023       for (fx = 0; fx < numXEdges; ++fx) {
1024         for (fz = 0; fz < numZVertices; fz++) {
1025           PetscInt face    = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
1026           PetscInt edgeL   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
1027           PetscInt edgeR   = firstYEdge + (fz * numXVertices + ((fx + 1) % numXVertices)) * numYEdges + fy;
1028           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1029           PetscInt edgeT   = firstXEdge + (fz * numYVertices + ((fy + 1) % numYVertices)) * numXEdges + fx;
1030           PetscInt ornt[4] = {0, 0, -1, -1};
1031           PetscInt cone[4];
1032 
1033           if (dim == 2) {
1034             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges - 1) {
1035               edgeR += numYEdges - 1 - 2 * fy;
1036               ornt[1] = -1;
1037             }
1038             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges - 1) {
1039               edgeT += numXEdges - 1 - 2 * fx;
1040               ornt[2] = 0;
1041             }
1042             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1043             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1044           } else {
1045             /* markers */
1046             if (bdZ != DM_BOUNDARY_PERIODIC) {
1047               if (fz == numZVertices - 1) {
1048                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop));
1049                 PetscCall(DMSetLabelValue(dm, "marker", face, markerTop));
1050               } else if (fz == 0) {
1051                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom));
1052                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBottom));
1053               }
1054             }
1055           }
1056           cone[0] = edgeB;
1057           cone[1] = edgeR;
1058           cone[2] = edgeT;
1059           cone[3] = edgeL;
1060           PetscCall(DMPlexSetCone(dm, face, cone));
1061           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1062         }
1063       }
1064     }
1065     /* Build Z edges*/
1066     for (vy = 0; vy < numYVertices; vy++) {
1067       for (vx = 0; vx < numXVertices; vx++) {
1068         for (ez = 0; ez < numZEdges; ez++) {
1069           const PetscInt edge    = firstZEdge + (vy * numXVertices + vx) * numZEdges + ez;
1070           const PetscInt vertexB = firstVertex + (ez * numYVertices + vy) * numXVertices + vx;
1071           const PetscInt vertexT = firstVertex + (((ez + 1) % numZVertices) * numYVertices + vy) * numXVertices + vx;
1072           PetscInt       cone[2];
1073 
1074           cone[0] = vertexB;
1075           cone[1] = vertexT;
1076           PetscCall(DMPlexSetCone(dm, edge, cone));
1077           if (dim == 3) {
1078             if (bdX != DM_BOUNDARY_PERIODIC) {
1079               if (vx == numXVertices - 1) {
1080                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1081                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1082                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1083               } else if (vx == 0) {
1084                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1085                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1086                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1087               }
1088             }
1089             if (bdY != DM_BOUNDARY_PERIODIC) {
1090               if (vy == numYVertices - 1) {
1091                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1092                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1093                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1094               } else if (vy == 0) {
1095                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1096                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1097                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1098               }
1099             }
1100           }
1101         }
1102       }
1103     }
1104     /* Build Y edges*/
1105     for (vz = 0; vz < numZVertices; vz++) {
1106       for (vx = 0; vx < numXVertices; vx++) {
1107         for (ey = 0; ey < numYEdges; ey++) {
1108           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges - 1) ? (numXVertices - vx - 1) : (vz * numYVertices + ((ey + 1) % numYVertices)) * numXVertices + vx;
1109           const PetscInt edge    = firstYEdge + (vz * numXVertices + vx) * numYEdges + ey;
1110           const PetscInt vertexF = firstVertex + (vz * numYVertices + ey) * numXVertices + vx;
1111           const PetscInt vertexK = firstVertex + nextv;
1112           PetscInt       cone[2];
1113 
1114           cone[0] = vertexF;
1115           cone[1] = vertexK;
1116           PetscCall(DMPlexSetCone(dm, edge, cone));
1117           if (dim == 2) {
1118             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1119               if (vx == numXVertices - 1) {
1120                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight));
1121                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1122                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1123                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1124               } else if (vx == 0) {
1125                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft));
1126                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1127                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1128                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1129               }
1130             } else {
1131               if (vx == 0 && cutLabel) {
1132                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1133                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1134                 if (ey == numYEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1135               }
1136             }
1137           } else {
1138             if (bdX != DM_BOUNDARY_PERIODIC) {
1139               if (vx == numXVertices - 1) {
1140                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1141                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1142                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1143               } else if (vx == 0) {
1144                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1145                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1146                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1147               }
1148             }
1149             if (bdZ != DM_BOUNDARY_PERIODIC) {
1150               if (vz == numZVertices - 1) {
1151                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1152                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1153                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1154               } else if (vz == 0) {
1155                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1156                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1157                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1158               }
1159             }
1160           }
1161         }
1162       }
1163     }
1164     /* Build X edges*/
1165     for (vz = 0; vz < numZVertices; vz++) {
1166       for (vy = 0; vy < numYVertices; vy++) {
1167         for (ex = 0; ex < numXEdges; ex++) {
1168           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges - 1) ? (numYVertices - vy - 1) * numXVertices : (vz * numYVertices + vy) * numXVertices + (ex + 1) % numXVertices;
1169           const PetscInt edge    = firstXEdge + (vz * numYVertices + vy) * numXEdges + ex;
1170           const PetscInt vertexL = firstVertex + (vz * numYVertices + vy) * numXVertices + ex;
1171           const PetscInt vertexR = firstVertex + nextv;
1172           PetscInt       cone[2];
1173 
1174           cone[0] = vertexL;
1175           cone[1] = vertexR;
1176           PetscCall(DMPlexSetCone(dm, edge, cone));
1177           if (dim == 2) {
1178             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1179               if (vy == numYVertices - 1) {
1180                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop));
1181                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1182                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1183                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1184               } else if (vy == 0) {
1185                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom));
1186                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1187                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1188                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1189               }
1190             } else {
1191               if (vy == 0 && cutLabel) {
1192                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1193                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1194                 if (ex == numXEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1195               }
1196             }
1197           } else {
1198             if (bdY != DM_BOUNDARY_PERIODIC) {
1199               if (vy == numYVertices - 1) {
1200                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1201                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1202                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1203               } else if (vy == 0) {
1204                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1205                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1206                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1207               }
1208             }
1209             if (bdZ != DM_BOUNDARY_PERIODIC) {
1210               if (vz == numZVertices - 1) {
1211                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1212                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1213                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1214               } else if (vz == 0) {
1215                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1216                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1217                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1218               }
1219             }
1220           }
1221         }
1222       }
1223     }
1224     PetscCall(DMPlexSymmetrize(dm));
1225     PetscCall(DMPlexStratify(dm));
1226     /* Build coordinates */
1227     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1228     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1229     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1230     PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVertices));
1231     for (v = firstVertex; v < firstVertex + numVertices; ++v) {
1232       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1233       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1234     }
1235     PetscCall(PetscSectionSetUp(coordSection));
1236     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1237     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1238     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1239     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1240     PetscCall(VecSetBlockSize(coordinates, dim));
1241     PetscCall(VecSetType(coordinates, VECSTANDARD));
1242     PetscCall(VecGetArray(coordinates, &coords));
1243     for (vz = 0; vz < numZVertices; ++vz) {
1244       for (vy = 0; vy < numYVertices; ++vy) {
1245         for (vx = 0; vx < numXVertices; ++vx) {
1246           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * vx;
1247           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * vy;
1248           if (dim == 3) coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 2] = lower[2] + ((upper[2] - lower[2]) / numZEdges) * vz;
1249         }
1250       }
1251     }
1252     PetscCall(VecRestoreArray(coordinates, &coords));
1253     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1254     PetscCall(VecDestroy(&coordinates));
1255   }
1256   PetscFunctionReturn(0);
1257 }
1258 
1259 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1260 {
1261   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1262   PetscInt       fac[3] = {0, 0, 0}, d;
1263 
1264   PetscFunctionBegin;
1265   PetscValidPointer(dm, 1);
1266   PetscValidLogicalCollectiveInt(dm, dim, 2);
1267   PetscCall(DMSetDimension(dm, dim));
1268   for (d = 0; d < dim; ++d) {
1269     fac[d] = faces[d];
1270     bdt[d] = periodicity[d];
1271   }
1272   PetscCall(DMPlexCreateCubeMesh_Internal(dm, lower, upper, fac, bdt[0], bdt[1], bdt[2]));
1273   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST || periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST || (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
1274     PetscReal L[3]       = {-1., -1., 0.};
1275     PetscReal maxCell[3] = {-1., -1., 0.};
1276 
1277     for (d = 0; d < dim; ++d) {
1278       if (periodicity[d] != DM_BOUNDARY_NONE) {
1279         L[d]       = upper[d] - lower[d];
1280         maxCell[d] = 1.1 * (L[d] / PetscMax(1, faces[d]));
1281       }
1282     }
1283     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1284   }
1285   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
1286   PetscFunctionReturn(0);
1287 }
1288 
1289 static PetscErrorCode DMPlexCreateBoxMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1290 {
1291   PetscFunctionBegin;
1292   if (dim == 1) PetscCall(DMPlexCreateLineMesh_Internal(dm, faces[0], lower[0], upper[0], periodicity[0]));
1293   else if (simplex) PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1294   else PetscCall(DMPlexCreateBoxMesh_Tensor_Internal(dm, dim, faces, lower, upper, periodicity));
1295   if (!interpolate && dim > 1 && !simplex) {
1296     DM udm;
1297 
1298     PetscCall(DMPlexUninterpolate(dm, &udm));
1299     PetscCall(DMPlexCopyCoordinates(dm, udm));
1300     PetscCall(DMPlexReplace_Internal(dm, &udm));
1301   }
1302   PetscFunctionReturn(0);
1303 }
1304 
1305 /*@C
1306   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
1307 
1308   Collective
1309 
1310   Input Parameters:
1311 + comm        - The communicator for the `DM` object
1312 . dim         - The spatial dimension
1313 . simplex     - `PETSC_TRUE` for simplices, `PETSC_FALSE` for tensor cells
1314 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1315 . lower       - The lower left corner, or NULL for (0, 0, 0)
1316 . upper       - The upper right corner, or NULL for (1, 1, 1)
1317 . periodicity - The boundary type for the X,Y,Z direction, or NULL for `DM_BOUNDARY_NONE`
1318 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1319 
1320   Output Parameter:
1321 . dm  - The `DM` object
1322 
1323   Level: beginner
1324 
1325   Note:
1326    To customize this mesh using options, use
1327 .vb
1328   DMCreate(comm, &dm);
1329   DMSetType(dm, DMPLEX);
1330   DMSetFromOptions(dm);
1331 .ve
1332 and use the options in `DMSetFromOptions()`.
1333 
1334   Here is the numbering returned for 2 faces in each direction for tensor cells:
1335 .vb
1336  10---17---11---18----12
1337   |         |         |
1338   |         |         |
1339  20    2   22    3    24
1340   |         |         |
1341   |         |         |
1342   7---15----8---16----9
1343   |         |         |
1344   |         |         |
1345  19    0   21    1   23
1346   |         |         |
1347   |         |         |
1348   4---13----5---14----6
1349 .ve
1350 and for simplicial cells
1351 .vb
1352  14----8---15----9----16
1353   |\     5  |\      7 |
1354   | \       | \       |
1355  13   2    14    3    15
1356   | 4   \   | 6   \   |
1357   |       \ |       \ |
1358  11----6---12----7----13
1359   |\        |\        |
1360   | \    1  | \     3 |
1361  10   0    11    1    12
1362   | 0   \   | 2   \   |
1363   |       \ |       \ |
1364   8----4----9----5----10
1365 .ve
1366 
1367 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
1368 @*/
1369 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
1370 {
1371   PetscInt       fac[3] = {1, 1, 1};
1372   PetscReal      low[3] = {0, 0, 0};
1373   PetscReal      upp[3] = {1, 1, 1};
1374   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1375 
1376   PetscFunctionBegin;
1377   PetscCall(DMCreate(comm, dm));
1378   PetscCall(DMSetType(*dm, DMPLEX));
1379   PetscCall(DMPlexCreateBoxMesh_Internal(*dm, dim, simplex, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt, interpolate));
1380   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1381   PetscFunctionReturn(0);
1382 }
1383 
1384 static PetscErrorCode DMPlexCreateWedgeBoxMesh_Internal(DM dm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1385 {
1386   DM       bdm, vol;
1387   PetscInt i;
1388 
1389   PetscFunctionBegin;
1390   for (i = 0; i < 3; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity not yet supported");
1391   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &bdm));
1392   PetscCall(DMSetType(bdm, DMPLEX));
1393   PetscCall(DMSetDimension(bdm, 2));
1394   PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(bdm, 2, faces, lower, upper, periodicity, PETSC_TRUE));
1395   PetscCall(DMPlexExtrude(bdm, faces[2], upper[2] - lower[2], PETSC_TRUE, PETSC_FALSE, NULL, NULL, &vol));
1396   PetscCall(DMDestroy(&bdm));
1397   PetscCall(DMPlexReplace_Internal(dm, &vol));
1398   if (lower[2] != 0.0) {
1399     Vec          v;
1400     PetscScalar *x;
1401     PetscInt     cDim, n;
1402 
1403     PetscCall(DMGetCoordinatesLocal(dm, &v));
1404     PetscCall(VecGetBlockSize(v, &cDim));
1405     PetscCall(VecGetLocalSize(v, &n));
1406     PetscCall(VecGetArray(v, &x));
1407     x += cDim;
1408     for (i = 0; i < n; i += cDim) x[i] += lower[2];
1409     PetscCall(VecRestoreArray(v, &x));
1410     PetscCall(DMSetCoordinatesLocal(dm, v));
1411   }
1412   PetscFunctionReturn(0);
1413 }
1414 
1415 /*@
1416   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tesselating the (x,y) plane and extruding in the third direction using wedge cells.
1417 
1418   Collective
1419 
1420   Input Parameters:
1421 + comm        - The communicator for the `DM` object
1422 . faces       - Number of faces per dimension, or NULL for (1, 1, 1)
1423 . lower       - The lower left corner, or NULL for (0, 0, 0)
1424 . upper       - The upper right corner, or NULL for (1, 1, 1)
1425 . periodicity - The boundary type for the X,Y,Z direction, or NULL for `DM_BOUNDARY_NONE`
1426 . orderHeight - If `PETSC_TRUE`, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1427 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1428 
1429   Output Parameter:
1430 . dm  - The `DM` object
1431 
1432   Level: beginner
1433 
1434 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateWedgeCylinderMesh()`, `DMExtrude()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1435 @*/
1436 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool orderHeight, PetscBool interpolate, DM *dm)
1437 {
1438   PetscInt       fac[3] = {1, 1, 1};
1439   PetscReal      low[3] = {0, 0, 0};
1440   PetscReal      upp[3] = {1, 1, 1};
1441   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1442 
1443   PetscFunctionBegin;
1444   PetscCall(DMCreate(comm, dm));
1445   PetscCall(DMSetType(*dm, DMPLEX));
1446   PetscCall(DMPlexCreateWedgeBoxMesh_Internal(*dm, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt));
1447   if (!interpolate) {
1448     DM udm;
1449 
1450     PetscCall(DMPlexUninterpolate(*dm, &udm));
1451     PetscCall(DMPlexReplace_Internal(*dm, &udm));
1452   }
1453   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1454   PetscFunctionReturn(0);
1455 }
1456 
1457 /*@C
1458   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all `DM` options in the database.
1459 
1460   Logically Collective on dm
1461 
1462   Input Parameters:
1463 + dm - the DM context
1464 - prefix - the prefix to prepend to all option names
1465 
1466   Level: advanced
1467 
1468   Note:
1469   A hyphen (-) must NOT be given at the beginning of the prefix name.
1470   The first character of all runtime options is AUTOMATICALLY the hyphen.
1471 
1472 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `SNESSetFromOptions()`
1473 @*/
1474 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
1475 {
1476   DM_Plex *mesh = (DM_Plex *)dm->data;
1477 
1478   PetscFunctionBegin;
1479   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1480   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, prefix));
1481   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)mesh->partitioner, prefix));
1482   PetscFunctionReturn(0);
1483 }
1484 
1485 /* Remap geometry to cylinder
1486    TODO: This only works for a single refinement, then it is broken
1487 
1488      Interior square: Linear interpolation is correct
1489      The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
1490      such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
1491 
1492        phi     = arctan(y/x)
1493        d_close = sqrt(1/8 + 1/4 sin^2(phi))
1494        d_far   = sqrt(1/2 + sin^2(phi))
1495 
1496      so we remap them using
1497 
1498        x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
1499        y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
1500 
1501      If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
1502 */
1503 static void snapToCylinder(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
1504 {
1505   const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1506   const PetscReal ds2 = 0.5 * dis;
1507 
1508   if ((PetscAbsScalar(u[0]) <= ds2) && (PetscAbsScalar(u[1]) <= ds2)) {
1509     f0[0] = u[0];
1510     f0[1] = u[1];
1511   } else {
1512     PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
1513 
1514     x    = PetscRealPart(u[0]);
1515     y    = PetscRealPart(u[1]);
1516     phi  = PetscAtan2Real(y, x);
1517     sinp = PetscSinReal(phi);
1518     cosp = PetscCosReal(phi);
1519     if ((PetscAbsReal(phi) > PETSC_PI / 4.0) && (PetscAbsReal(phi) < 3.0 * PETSC_PI / 4.0)) {
1520       dc = PetscAbsReal(ds2 / sinp);
1521       df = PetscAbsReal(dis / sinp);
1522       xc = ds2 * x / PetscAbsReal(y);
1523       yc = ds2 * PetscSignReal(y);
1524     } else {
1525       dc = PetscAbsReal(ds2 / cosp);
1526       df = PetscAbsReal(dis / cosp);
1527       xc = ds2 * PetscSignReal(x);
1528       yc = ds2 * y / PetscAbsReal(x);
1529     }
1530     f0[0] = xc + (u[0] - xc) * (1.0 - dc) / (df - dc);
1531     f0[1] = yc + (u[1] - yc) * (1.0 - dc) / (df - dc);
1532   }
1533   f0[2] = u[2];
1534 }
1535 
1536 static PetscErrorCode DMPlexCreateHexCylinderMesh_Internal(DM dm, DMBoundaryType periodicZ)
1537 {
1538   const PetscInt dim = 3;
1539   PetscInt       numCells, numVertices;
1540   PetscMPIInt    rank;
1541 
1542   PetscFunctionBegin;
1543   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1544   PetscCall(DMSetDimension(dm, dim));
1545   /* Create topology */
1546   {
1547     PetscInt cone[8], c;
1548 
1549     numCells    = rank == 0 ? 5 : 0;
1550     numVertices = rank == 0 ? 16 : 0;
1551     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1552       numCells *= 3;
1553       numVertices = rank == 0 ? 24 : 0;
1554     }
1555     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1556     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 8));
1557     PetscCall(DMSetUp(dm));
1558     if (rank == 0) {
1559       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1560         cone[0] = 15;
1561         cone[1] = 18;
1562         cone[2] = 17;
1563         cone[3] = 16;
1564         cone[4] = 31;
1565         cone[5] = 32;
1566         cone[6] = 33;
1567         cone[7] = 34;
1568         PetscCall(DMPlexSetCone(dm, 0, cone));
1569         cone[0] = 16;
1570         cone[1] = 17;
1571         cone[2] = 24;
1572         cone[3] = 23;
1573         cone[4] = 32;
1574         cone[5] = 36;
1575         cone[6] = 37;
1576         cone[7] = 33; /* 22 25 26 21 */
1577         PetscCall(DMPlexSetCone(dm, 1, cone));
1578         cone[0] = 18;
1579         cone[1] = 27;
1580         cone[2] = 24;
1581         cone[3] = 17;
1582         cone[4] = 34;
1583         cone[5] = 33;
1584         cone[6] = 37;
1585         cone[7] = 38;
1586         PetscCall(DMPlexSetCone(dm, 2, cone));
1587         cone[0] = 29;
1588         cone[1] = 27;
1589         cone[2] = 18;
1590         cone[3] = 15;
1591         cone[4] = 35;
1592         cone[5] = 31;
1593         cone[6] = 34;
1594         cone[7] = 38;
1595         PetscCall(DMPlexSetCone(dm, 3, cone));
1596         cone[0] = 29;
1597         cone[1] = 15;
1598         cone[2] = 16;
1599         cone[3] = 23;
1600         cone[4] = 35;
1601         cone[5] = 36;
1602         cone[6] = 32;
1603         cone[7] = 31;
1604         PetscCall(DMPlexSetCone(dm, 4, cone));
1605 
1606         cone[0] = 31;
1607         cone[1] = 34;
1608         cone[2] = 33;
1609         cone[3] = 32;
1610         cone[4] = 19;
1611         cone[5] = 22;
1612         cone[6] = 21;
1613         cone[7] = 20;
1614         PetscCall(DMPlexSetCone(dm, 5, cone));
1615         cone[0] = 32;
1616         cone[1] = 33;
1617         cone[2] = 37;
1618         cone[3] = 36;
1619         cone[4] = 22;
1620         cone[5] = 25;
1621         cone[6] = 26;
1622         cone[7] = 21;
1623         PetscCall(DMPlexSetCone(dm, 6, cone));
1624         cone[0] = 34;
1625         cone[1] = 38;
1626         cone[2] = 37;
1627         cone[3] = 33;
1628         cone[4] = 20;
1629         cone[5] = 21;
1630         cone[6] = 26;
1631         cone[7] = 28;
1632         PetscCall(DMPlexSetCone(dm, 7, cone));
1633         cone[0] = 35;
1634         cone[1] = 38;
1635         cone[2] = 34;
1636         cone[3] = 31;
1637         cone[4] = 30;
1638         cone[5] = 19;
1639         cone[6] = 20;
1640         cone[7] = 28;
1641         PetscCall(DMPlexSetCone(dm, 8, cone));
1642         cone[0] = 35;
1643         cone[1] = 31;
1644         cone[2] = 32;
1645         cone[3] = 36;
1646         cone[4] = 30;
1647         cone[5] = 25;
1648         cone[6] = 22;
1649         cone[7] = 19;
1650         PetscCall(DMPlexSetCone(dm, 9, cone));
1651 
1652         cone[0] = 19;
1653         cone[1] = 20;
1654         cone[2] = 21;
1655         cone[3] = 22;
1656         cone[4] = 15;
1657         cone[5] = 16;
1658         cone[6] = 17;
1659         cone[7] = 18;
1660         PetscCall(DMPlexSetCone(dm, 10, cone));
1661         cone[0] = 22;
1662         cone[1] = 21;
1663         cone[2] = 26;
1664         cone[3] = 25;
1665         cone[4] = 16;
1666         cone[5] = 23;
1667         cone[6] = 24;
1668         cone[7] = 17;
1669         PetscCall(DMPlexSetCone(dm, 11, cone));
1670         cone[0] = 20;
1671         cone[1] = 28;
1672         cone[2] = 26;
1673         cone[3] = 21;
1674         cone[4] = 18;
1675         cone[5] = 17;
1676         cone[6] = 24;
1677         cone[7] = 27;
1678         PetscCall(DMPlexSetCone(dm, 12, cone));
1679         cone[0] = 30;
1680         cone[1] = 28;
1681         cone[2] = 20;
1682         cone[3] = 19;
1683         cone[4] = 29;
1684         cone[5] = 15;
1685         cone[6] = 18;
1686         cone[7] = 27;
1687         PetscCall(DMPlexSetCone(dm, 13, cone));
1688         cone[0] = 30;
1689         cone[1] = 19;
1690         cone[2] = 22;
1691         cone[3] = 25;
1692         cone[4] = 29;
1693         cone[5] = 23;
1694         cone[6] = 16;
1695         cone[7] = 15;
1696         PetscCall(DMPlexSetCone(dm, 14, cone));
1697       } else {
1698         cone[0] = 5;
1699         cone[1] = 8;
1700         cone[2] = 7;
1701         cone[3] = 6;
1702         cone[4] = 9;
1703         cone[5] = 12;
1704         cone[6] = 11;
1705         cone[7] = 10;
1706         PetscCall(DMPlexSetCone(dm, 0, cone));
1707         cone[0] = 6;
1708         cone[1] = 7;
1709         cone[2] = 14;
1710         cone[3] = 13;
1711         cone[4] = 12;
1712         cone[5] = 15;
1713         cone[6] = 16;
1714         cone[7] = 11;
1715         PetscCall(DMPlexSetCone(dm, 1, cone));
1716         cone[0] = 8;
1717         cone[1] = 17;
1718         cone[2] = 14;
1719         cone[3] = 7;
1720         cone[4] = 10;
1721         cone[5] = 11;
1722         cone[6] = 16;
1723         cone[7] = 18;
1724         PetscCall(DMPlexSetCone(dm, 2, cone));
1725         cone[0] = 19;
1726         cone[1] = 17;
1727         cone[2] = 8;
1728         cone[3] = 5;
1729         cone[4] = 20;
1730         cone[5] = 9;
1731         cone[6] = 10;
1732         cone[7] = 18;
1733         PetscCall(DMPlexSetCone(dm, 3, cone));
1734         cone[0] = 19;
1735         cone[1] = 5;
1736         cone[2] = 6;
1737         cone[3] = 13;
1738         cone[4] = 20;
1739         cone[5] = 15;
1740         cone[6] = 12;
1741         cone[7] = 9;
1742         PetscCall(DMPlexSetCone(dm, 4, cone));
1743       }
1744     }
1745     PetscCall(DMPlexSymmetrize(dm));
1746     PetscCall(DMPlexStratify(dm));
1747   }
1748   /* Create cube geometry */
1749   {
1750     Vec             coordinates;
1751     PetscSection    coordSection;
1752     PetscScalar    *coords;
1753     PetscInt        coordSize, v;
1754     const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1755     const PetscReal ds2 = dis / 2.0;
1756 
1757     /* Build coordinates */
1758     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1759     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1760     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1761     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1762     for (v = numCells; v < numCells + numVertices; ++v) {
1763       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1764       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1765     }
1766     PetscCall(PetscSectionSetUp(coordSection));
1767     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1768     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1769     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1770     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1771     PetscCall(VecSetBlockSize(coordinates, dim));
1772     PetscCall(VecSetType(coordinates, VECSTANDARD));
1773     PetscCall(VecGetArray(coordinates, &coords));
1774     if (rank == 0) {
1775       coords[0 * dim + 0]  = -ds2;
1776       coords[0 * dim + 1]  = -ds2;
1777       coords[0 * dim + 2]  = 0.0;
1778       coords[1 * dim + 0]  = ds2;
1779       coords[1 * dim + 1]  = -ds2;
1780       coords[1 * dim + 2]  = 0.0;
1781       coords[2 * dim + 0]  = ds2;
1782       coords[2 * dim + 1]  = ds2;
1783       coords[2 * dim + 2]  = 0.0;
1784       coords[3 * dim + 0]  = -ds2;
1785       coords[3 * dim + 1]  = ds2;
1786       coords[3 * dim + 2]  = 0.0;
1787       coords[4 * dim + 0]  = -ds2;
1788       coords[4 * dim + 1]  = -ds2;
1789       coords[4 * dim + 2]  = 1.0;
1790       coords[5 * dim + 0]  = -ds2;
1791       coords[5 * dim + 1]  = ds2;
1792       coords[5 * dim + 2]  = 1.0;
1793       coords[6 * dim + 0]  = ds2;
1794       coords[6 * dim + 1]  = ds2;
1795       coords[6 * dim + 2]  = 1.0;
1796       coords[7 * dim + 0]  = ds2;
1797       coords[7 * dim + 1]  = -ds2;
1798       coords[7 * dim + 2]  = 1.0;
1799       coords[8 * dim + 0]  = dis;
1800       coords[8 * dim + 1]  = -dis;
1801       coords[8 * dim + 2]  = 0.0;
1802       coords[9 * dim + 0]  = dis;
1803       coords[9 * dim + 1]  = dis;
1804       coords[9 * dim + 2]  = 0.0;
1805       coords[10 * dim + 0] = dis;
1806       coords[10 * dim + 1] = -dis;
1807       coords[10 * dim + 2] = 1.0;
1808       coords[11 * dim + 0] = dis;
1809       coords[11 * dim + 1] = dis;
1810       coords[11 * dim + 2] = 1.0;
1811       coords[12 * dim + 0] = -dis;
1812       coords[12 * dim + 1] = dis;
1813       coords[12 * dim + 2] = 0.0;
1814       coords[13 * dim + 0] = -dis;
1815       coords[13 * dim + 1] = dis;
1816       coords[13 * dim + 2] = 1.0;
1817       coords[14 * dim + 0] = -dis;
1818       coords[14 * dim + 1] = -dis;
1819       coords[14 * dim + 2] = 0.0;
1820       coords[15 * dim + 0] = -dis;
1821       coords[15 * dim + 1] = -dis;
1822       coords[15 * dim + 2] = 1.0;
1823       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1824         /* 15 31 19 */ coords[16 * dim + 0] = -ds2;
1825         coords[16 * dim + 1]                = -ds2;
1826         coords[16 * dim + 2]                = 0.5;
1827         /* 16 32 22 */ coords[17 * dim + 0] = ds2;
1828         coords[17 * dim + 1]                = -ds2;
1829         coords[17 * dim + 2]                = 0.5;
1830         /* 17 33 21 */ coords[18 * dim + 0] = ds2;
1831         coords[18 * dim + 1]                = ds2;
1832         coords[18 * dim + 2]                = 0.5;
1833         /* 18 34 20 */ coords[19 * dim + 0] = -ds2;
1834         coords[19 * dim + 1]                = ds2;
1835         coords[19 * dim + 2]                = 0.5;
1836         /* 29 35 30 */ coords[20 * dim + 0] = -dis;
1837         coords[20 * dim + 1]                = -dis;
1838         coords[20 * dim + 2]                = 0.5;
1839         /* 23 36 25 */ coords[21 * dim + 0] = dis;
1840         coords[21 * dim + 1]                = -dis;
1841         coords[21 * dim + 2]                = 0.5;
1842         /* 24 37 26 */ coords[22 * dim + 0] = dis;
1843         coords[22 * dim + 1]                = dis;
1844         coords[22 * dim + 2]                = 0.5;
1845         /* 27 38 28 */ coords[23 * dim + 0] = -dis;
1846         coords[23 * dim + 1]                = dis;
1847         coords[23 * dim + 2]                = 0.5;
1848       }
1849     }
1850     PetscCall(VecRestoreArray(coordinates, &coords));
1851     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1852     PetscCall(VecDestroy(&coordinates));
1853   }
1854   /* Create periodicity */
1855   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1856     PetscReal L[3]       = {-1., -1., 0.};
1857     PetscReal maxCell[3] = {-1., -1., 0.};
1858     PetscReal lower[3]   = {0.0, 0.0, 0.0};
1859     PetscReal upper[3]   = {1.0, 1.0, 1.5};
1860     PetscInt  numZCells  = 3;
1861 
1862     L[2]       = upper[2] - lower[2];
1863     maxCell[2] = 1.1 * (L[2] / numZCells);
1864     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1865   }
1866   {
1867     DM          cdm;
1868     PetscDS     cds;
1869     PetscScalar c[2] = {1.0, 1.0};
1870 
1871     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToCylinder));
1872     PetscCall(DMGetCoordinateDM(dm, &cdm));
1873     PetscCall(DMGetDS(cdm, &cds));
1874     PetscCall(PetscDSSetConstants(cds, 2, c));
1875   }
1876   /* Wait for coordinate creation before doing in-place modification */
1877   PetscCall(DMPlexInterpolateInPlace_Internal(dm));
1878   PetscFunctionReturn(0);
1879 }
1880 
1881 /*@
1882   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
1883 
1884   Collective
1885 
1886   Input Parameters:
1887 + comm      - The communicator for the `DM` object
1888 - periodicZ - The boundary type for the Z direction
1889 
1890   Output Parameter:
1891 . dm  - The DM object
1892 
1893   Level: beginner
1894 
1895   Note:
1896   Here is the output numbering looking from the bottom of the cylinder:
1897 .vb
1898        17-----14
1899         |     |
1900         |  2  |
1901         |     |
1902  17-----8-----7-----14
1903   |     |     |     |
1904   |  3  |  0  |  1  |
1905   |     |     |     |
1906  19-----5-----6-----13
1907         |     |
1908         |  4  |
1909         |     |
1910        19-----13
1911 
1912  and up through the top
1913 
1914        18-----16
1915         |     |
1916         |  2  |
1917         |     |
1918  18----10----11-----16
1919   |     |     |     |
1920   |  3  |  0  |  1  |
1921   |     |     |     |
1922  20-----9----12-----15
1923         |     |
1924         |  4  |
1925         |     |
1926        20-----15
1927 .ve
1928 
1929 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1930 @*/
1931 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, DMBoundaryType periodicZ, DM *dm)
1932 {
1933   PetscFunctionBegin;
1934   PetscValidPointer(dm, 3);
1935   PetscCall(DMCreate(comm, dm));
1936   PetscCall(DMSetType(*dm, DMPLEX));
1937   PetscCall(DMPlexCreateHexCylinderMesh_Internal(*dm, periodicZ));
1938   PetscFunctionReturn(0);
1939 }
1940 
1941 static PetscErrorCode DMPlexCreateWedgeCylinderMesh_Internal(DM dm, PetscInt n, PetscBool interpolate)
1942 {
1943   const PetscInt dim = 3;
1944   PetscInt       numCells, numVertices, v;
1945   PetscMPIInt    rank;
1946 
1947   PetscFunctionBegin;
1948   PetscCheck(n >= 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %" PetscInt_FMT " cannot be negative", n);
1949   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1950   PetscCall(DMSetDimension(dm, dim));
1951   /* Must create the celltype label here so that we do not automatically try to compute the types */
1952   PetscCall(DMCreateLabel(dm, "celltype"));
1953   /* Create topology */
1954   {
1955     PetscInt cone[6], c;
1956 
1957     numCells    = rank == 0 ? n : 0;
1958     numVertices = rank == 0 ? 2 * (n + 1) : 0;
1959     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1960     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
1961     PetscCall(DMSetUp(dm));
1962     for (c = 0; c < numCells; c++) {
1963       cone[0] = c + n * 1;
1964       cone[1] = (c + 1) % n + n * 1;
1965       cone[2] = 0 + 3 * n;
1966       cone[3] = c + n * 2;
1967       cone[4] = (c + 1) % n + n * 2;
1968       cone[5] = 1 + 3 * n;
1969       PetscCall(DMPlexSetCone(dm, c, cone));
1970       PetscCall(DMPlexSetCellType(dm, c, DM_POLYTOPE_TRI_PRISM_TENSOR));
1971     }
1972     PetscCall(DMPlexSymmetrize(dm));
1973     PetscCall(DMPlexStratify(dm));
1974   }
1975   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetCellType(dm, v, DM_POLYTOPE_POINT));
1976   /* Create cylinder geometry */
1977   {
1978     Vec          coordinates;
1979     PetscSection coordSection;
1980     PetscScalar *coords;
1981     PetscInt     coordSize, c;
1982 
1983     /* Build coordinates */
1984     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1985     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1986     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1987     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1988     for (v = numCells; v < numCells + numVertices; ++v) {
1989       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1990       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1991     }
1992     PetscCall(PetscSectionSetUp(coordSection));
1993     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1994     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1995     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1996     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1997     PetscCall(VecSetBlockSize(coordinates, dim));
1998     PetscCall(VecSetType(coordinates, VECSTANDARD));
1999     PetscCall(VecGetArray(coordinates, &coords));
2000     for (c = 0; c < numCells; c++) {
2001       coords[(c + 0 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
2002       coords[(c + 0 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
2003       coords[(c + 0 * n) * dim + 2] = 1.0;
2004       coords[(c + 1 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
2005       coords[(c + 1 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
2006       coords[(c + 1 * n) * dim + 2] = 0.0;
2007     }
2008     if (rank == 0) {
2009       coords[(2 * n + 0) * dim + 0] = 0.0;
2010       coords[(2 * n + 0) * dim + 1] = 0.0;
2011       coords[(2 * n + 0) * dim + 2] = 1.0;
2012       coords[(2 * n + 1) * dim + 0] = 0.0;
2013       coords[(2 * n + 1) * dim + 1] = 0.0;
2014       coords[(2 * n + 1) * dim + 2] = 0.0;
2015     }
2016     PetscCall(VecRestoreArray(coordinates, &coords));
2017     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2018     PetscCall(VecDestroy(&coordinates));
2019   }
2020   /* Interpolate */
2021   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2022   PetscFunctionReturn(0);
2023 }
2024 
2025 /*@
2026   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
2027 
2028   Collective
2029 
2030   Input Parameters:
2031 + comm - The communicator for the `DM` object
2032 . n    - The number of wedges around the origin
2033 - interpolate - Create edges and faces
2034 
2035   Output Parameter:
2036 . dm  - The `DM` object
2037 
2038   Level: beginner
2039 
2040 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2041 @*/
2042 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
2043 {
2044   PetscFunctionBegin;
2045   PetscValidPointer(dm, 4);
2046   PetscCall(DMCreate(comm, dm));
2047   PetscCall(DMSetType(*dm, DMPLEX));
2048   PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(*dm, n, interpolate));
2049   PetscFunctionReturn(0);
2050 }
2051 
2052 static inline PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2053 {
2054   PetscReal prod = 0.0;
2055   PetscInt  i;
2056   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
2057   return PetscSqrtReal(prod);
2058 }
2059 static inline PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2060 {
2061   PetscReal prod = 0.0;
2062   PetscInt  i;
2063   for (i = 0; i < dim; ++i) prod += x[i] * y[i];
2064   return prod;
2065 }
2066 
2067 /* The first constant is the sphere radius */
2068 static void snapToSphere(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
2069 {
2070   PetscReal r     = PetscRealPart(constants[0]);
2071   PetscReal norm2 = 0.0, fac;
2072   PetscInt  n     = uOff[1] - uOff[0], d;
2073 
2074   for (d = 0; d < n; ++d) norm2 += PetscSqr(PetscRealPart(u[d]));
2075   fac = r / PetscSqrtReal(norm2);
2076   for (d = 0; d < n; ++d) f0[d] = u[d] * fac;
2077 }
2078 
2079 static PetscErrorCode DMPlexCreateSphereMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, PetscReal R)
2080 {
2081   const PetscInt embedDim = dim + 1;
2082   PetscSection   coordSection;
2083   Vec            coordinates;
2084   PetscScalar   *coords;
2085   PetscReal     *coordsIn;
2086   PetscInt       numCells, numEdges, numVerts = 0, firstVertex = 0, v, firstEdge, coordSize, d, c, e;
2087   PetscMPIInt    rank;
2088 
2089   PetscFunctionBegin;
2090   PetscValidLogicalCollectiveBool(dm, simplex, 3);
2091   PetscCall(DMSetDimension(dm, dim));
2092   PetscCall(DMSetCoordinateDim(dm, dim + 1));
2093   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2094   switch (dim) {
2095   case 2:
2096     if (simplex) {
2097       const PetscReal radius    = PetscSqrtReal(1 + PETSC_PHI * PETSC_PHI) / (1.0 + PETSC_PHI);
2098       const PetscReal edgeLen   = 2.0 / (1.0 + PETSC_PHI) * (R / radius);
2099       const PetscInt  degree    = 5;
2100       PetscReal       vertex[3] = {0.0, 1.0 / (1.0 + PETSC_PHI), PETSC_PHI / (1.0 + PETSC_PHI)};
2101       PetscInt        s[3]      = {1, 1, 1};
2102       PetscInt        cone[3];
2103       PetscInt       *graph, p, i, j, k;
2104 
2105       vertex[0] *= R / radius;
2106       vertex[1] *= R / radius;
2107       vertex[2] *= R / radius;
2108       numCells    = rank == 0 ? 20 : 0;
2109       numVerts    = rank == 0 ? 12 : 0;
2110       firstVertex = numCells;
2111       /* Use icosahedron, which for a R-sphere has coordinates which are all cyclic permutations of
2112 
2113            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
2114 
2115          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2116          length is then given by 2/(1+\phi) = 2 * 0.38197 = 0.76393.
2117       */
2118       /* Construct vertices */
2119       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2120       if (rank == 0) {
2121         for (p = 0, i = 0; p < embedDim; ++p) {
2122           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2123             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2124               for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertex[(d + p) % embedDim];
2125               ++i;
2126             }
2127           }
2128         }
2129       }
2130       /* Construct graph */
2131       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2132       for (i = 0; i < numVerts; ++i) {
2133         for (j = 0, k = 0; j < numVerts; ++j) {
2134           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2135             graph[i * numVerts + j] = 1;
2136             ++k;
2137           }
2138         }
2139         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid icosahedron, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2140       }
2141       /* Build Topology */
2142       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2143       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2144       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2145       /* Cells */
2146       for (i = 0, c = 0; i < numVerts; ++i) {
2147         for (j = 0; j < i; ++j) {
2148           for (k = 0; k < j; ++k) {
2149             if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i]) {
2150               cone[0] = firstVertex + i;
2151               cone[1] = firstVertex + j;
2152               cone[2] = firstVertex + k;
2153               /* Check orientation */
2154               {
2155                 const PetscInt epsilon[3][3][3] = {
2156                   {{0, 0, 0},  {0, 0, 1},  {0, -1, 0}},
2157                   {{0, 0, -1}, {0, 0, 0},  {1, 0, 0} },
2158                   {{0, 1, 0},  {-1, 0, 0}, {0, 0, 0} }
2159                 };
2160                 PetscReal normal[3];
2161                 PetscInt  e, f;
2162 
2163                 for (d = 0; d < embedDim; ++d) {
2164                   normal[d] = 0.0;
2165                   for (e = 0; e < embedDim; ++e) {
2166                     for (f = 0; f < embedDim; ++f) normal[d] += epsilon[d][e][f] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]);
2167                   }
2168                 }
2169                 if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2170                   PetscInt tmp = cone[1];
2171                   cone[1]      = cone[2];
2172                   cone[2]      = tmp;
2173                 }
2174               }
2175               PetscCall(DMPlexSetCone(dm, c++, cone));
2176             }
2177           }
2178         }
2179       }
2180       PetscCall(DMPlexSymmetrize(dm));
2181       PetscCall(DMPlexStratify(dm));
2182       PetscCall(PetscFree(graph));
2183     } else {
2184       /*
2185         12-21--13
2186          |     |
2187         25  4  24
2188          |     |
2189   12-25--9-16--8-24--13
2190    |     |     |     |
2191   23  5 17  0 15  3  22
2192    |     |     |     |
2193   10-20--6-14--7-19--11
2194          |     |
2195         20  1  19
2196          |     |
2197         10-18--11
2198          |     |
2199         23  2  22
2200          |     |
2201         12-21--13
2202        */
2203       PetscInt cone[4], ornt[4];
2204 
2205       numCells    = rank == 0 ? 6 : 0;
2206       numEdges    = rank == 0 ? 12 : 0;
2207       numVerts    = rank == 0 ? 8 : 0;
2208       firstVertex = numCells;
2209       firstEdge   = numCells + numVerts;
2210       /* Build Topology */
2211       PetscCall(DMPlexSetChart(dm, 0, numCells + numEdges + numVerts));
2212       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 4));
2213       for (e = firstEdge; e < firstEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
2214       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2215       if (rank == 0) {
2216         /* Cell 0 */
2217         cone[0] = 14;
2218         cone[1] = 15;
2219         cone[2] = 16;
2220         cone[3] = 17;
2221         PetscCall(DMPlexSetCone(dm, 0, cone));
2222         ornt[0] = 0;
2223         ornt[1] = 0;
2224         ornt[2] = 0;
2225         ornt[3] = 0;
2226         PetscCall(DMPlexSetConeOrientation(dm, 0, ornt));
2227         /* Cell 1 */
2228         cone[0] = 18;
2229         cone[1] = 19;
2230         cone[2] = 14;
2231         cone[3] = 20;
2232         PetscCall(DMPlexSetCone(dm, 1, cone));
2233         ornt[0] = 0;
2234         ornt[1] = 0;
2235         ornt[2] = -1;
2236         ornt[3] = 0;
2237         PetscCall(DMPlexSetConeOrientation(dm, 1, ornt));
2238         /* Cell 2 */
2239         cone[0] = 21;
2240         cone[1] = 22;
2241         cone[2] = 18;
2242         cone[3] = 23;
2243         PetscCall(DMPlexSetCone(dm, 2, cone));
2244         ornt[0] = 0;
2245         ornt[1] = 0;
2246         ornt[2] = -1;
2247         ornt[3] = 0;
2248         PetscCall(DMPlexSetConeOrientation(dm, 2, ornt));
2249         /* Cell 3 */
2250         cone[0] = 19;
2251         cone[1] = 22;
2252         cone[2] = 24;
2253         cone[3] = 15;
2254         PetscCall(DMPlexSetCone(dm, 3, cone));
2255         ornt[0] = -1;
2256         ornt[1] = -1;
2257         ornt[2] = 0;
2258         ornt[3] = -1;
2259         PetscCall(DMPlexSetConeOrientation(dm, 3, ornt));
2260         /* Cell 4 */
2261         cone[0] = 16;
2262         cone[1] = 24;
2263         cone[2] = 21;
2264         cone[3] = 25;
2265         PetscCall(DMPlexSetCone(dm, 4, cone));
2266         ornt[0] = -1;
2267         ornt[1] = -1;
2268         ornt[2] = -1;
2269         ornt[3] = 0;
2270         PetscCall(DMPlexSetConeOrientation(dm, 4, ornt));
2271         /* Cell 5 */
2272         cone[0] = 20;
2273         cone[1] = 17;
2274         cone[2] = 25;
2275         cone[3] = 23;
2276         PetscCall(DMPlexSetCone(dm, 5, cone));
2277         ornt[0] = -1;
2278         ornt[1] = -1;
2279         ornt[2] = -1;
2280         ornt[3] = -1;
2281         PetscCall(DMPlexSetConeOrientation(dm, 5, ornt));
2282         /* Edges */
2283         cone[0] = 6;
2284         cone[1] = 7;
2285         PetscCall(DMPlexSetCone(dm, 14, cone));
2286         cone[0] = 7;
2287         cone[1] = 8;
2288         PetscCall(DMPlexSetCone(dm, 15, cone));
2289         cone[0] = 8;
2290         cone[1] = 9;
2291         PetscCall(DMPlexSetCone(dm, 16, cone));
2292         cone[0] = 9;
2293         cone[1] = 6;
2294         PetscCall(DMPlexSetCone(dm, 17, cone));
2295         cone[0] = 10;
2296         cone[1] = 11;
2297         PetscCall(DMPlexSetCone(dm, 18, cone));
2298         cone[0] = 11;
2299         cone[1] = 7;
2300         PetscCall(DMPlexSetCone(dm, 19, cone));
2301         cone[0] = 6;
2302         cone[1] = 10;
2303         PetscCall(DMPlexSetCone(dm, 20, cone));
2304         cone[0] = 12;
2305         cone[1] = 13;
2306         PetscCall(DMPlexSetCone(dm, 21, cone));
2307         cone[0] = 13;
2308         cone[1] = 11;
2309         PetscCall(DMPlexSetCone(dm, 22, cone));
2310         cone[0] = 10;
2311         cone[1] = 12;
2312         PetscCall(DMPlexSetCone(dm, 23, cone));
2313         cone[0] = 13;
2314         cone[1] = 8;
2315         PetscCall(DMPlexSetCone(dm, 24, cone));
2316         cone[0] = 12;
2317         cone[1] = 9;
2318         PetscCall(DMPlexSetCone(dm, 25, cone));
2319       }
2320       PetscCall(DMPlexSymmetrize(dm));
2321       PetscCall(DMPlexStratify(dm));
2322       /* Build coordinates */
2323       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2324       if (rank == 0) {
2325         coordsIn[0 * embedDim + 0] = -R;
2326         coordsIn[0 * embedDim + 1] = R;
2327         coordsIn[0 * embedDim + 2] = -R;
2328         coordsIn[1 * embedDim + 0] = R;
2329         coordsIn[1 * embedDim + 1] = R;
2330         coordsIn[1 * embedDim + 2] = -R;
2331         coordsIn[2 * embedDim + 0] = R;
2332         coordsIn[2 * embedDim + 1] = -R;
2333         coordsIn[2 * embedDim + 2] = -R;
2334         coordsIn[3 * embedDim + 0] = -R;
2335         coordsIn[3 * embedDim + 1] = -R;
2336         coordsIn[3 * embedDim + 2] = -R;
2337         coordsIn[4 * embedDim + 0] = -R;
2338         coordsIn[4 * embedDim + 1] = R;
2339         coordsIn[4 * embedDim + 2] = R;
2340         coordsIn[5 * embedDim + 0] = R;
2341         coordsIn[5 * embedDim + 1] = R;
2342         coordsIn[5 * embedDim + 2] = R;
2343         coordsIn[6 * embedDim + 0] = -R;
2344         coordsIn[6 * embedDim + 1] = -R;
2345         coordsIn[6 * embedDim + 2] = R;
2346         coordsIn[7 * embedDim + 0] = R;
2347         coordsIn[7 * embedDim + 1] = -R;
2348         coordsIn[7 * embedDim + 2] = R;
2349       }
2350     }
2351     break;
2352   case 3:
2353     if (simplex) {
2354       const PetscReal edgeLen         = 1.0 / PETSC_PHI;
2355       PetscReal       vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
2356       PetscReal       vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
2357       PetscReal       vertexC[4]      = {0.5, 0.5 * PETSC_PHI, 0.5 / PETSC_PHI, 0.0};
2358       const PetscInt  degree          = 12;
2359       PetscInt        s[4]            = {1, 1, 1};
2360       PetscInt        evenPerm[12][4] = {
2361         {0, 1, 2, 3},
2362         {0, 2, 3, 1},
2363         {0, 3, 1, 2},
2364         {1, 0, 3, 2},
2365         {1, 2, 0, 3},
2366         {1, 3, 2, 0},
2367         {2, 0, 1, 3},
2368         {2, 1, 3, 0},
2369         {2, 3, 0, 1},
2370         {3, 0, 2, 1},
2371         {3, 1, 0, 2},
2372         {3, 2, 1, 0}
2373       };
2374       PetscInt  cone[4];
2375       PetscInt *graph, p, i, j, k, l;
2376 
2377       vertexA[0] *= R;
2378       vertexA[1] *= R;
2379       vertexA[2] *= R;
2380       vertexA[3] *= R;
2381       vertexB[0] *= R;
2382       vertexB[1] *= R;
2383       vertexB[2] *= R;
2384       vertexB[3] *= R;
2385       vertexC[0] *= R;
2386       vertexC[1] *= R;
2387       vertexC[2] *= R;
2388       vertexC[3] *= R;
2389       numCells    = rank == 0 ? 600 : 0;
2390       numVerts    = rank == 0 ? 120 : 0;
2391       firstVertex = numCells;
2392       /* Use the 600-cell, which for a unit sphere has coordinates which are
2393 
2394            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
2395                (\pm 1,    0,       0,      0)  all cyclic permutations   8
2396            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
2397 
2398          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2399          length is then given by 1/\phi = 0.61803.
2400 
2401          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
2402          http://mathworld.wolfram.com/600-Cell.html
2403       */
2404       /* Construct vertices */
2405       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2406       i = 0;
2407       if (rank == 0) {
2408         for (s[0] = -1; s[0] < 2; s[0] += 2) {
2409           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2410             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2411               for (s[3] = -1; s[3] < 2; s[3] += 2) {
2412                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[d] * vertexA[d];
2413                 ++i;
2414               }
2415             }
2416           }
2417         }
2418         for (p = 0; p < embedDim; ++p) {
2419           s[1] = s[2] = s[3] = 1;
2420           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2421             for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertexB[(d + p) % embedDim];
2422             ++i;
2423           }
2424         }
2425         for (p = 0; p < 12; ++p) {
2426           s[3] = 1;
2427           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2428             for (s[1] = -1; s[1] < 2; s[1] += 2) {
2429               for (s[2] = -1; s[2] < 2; s[2] += 2) {
2430                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[evenPerm[p][d]] * vertexC[evenPerm[p][d]];
2431                 ++i;
2432               }
2433             }
2434           }
2435         }
2436       }
2437       PetscCheck(i == numVerts, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertices %" PetscInt_FMT " != %" PetscInt_FMT, i, numVerts);
2438       /* Construct graph */
2439       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2440       for (i = 0; i < numVerts; ++i) {
2441         for (j = 0, k = 0; j < numVerts; ++j) {
2442           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2443             graph[i * numVerts + j] = 1;
2444             ++k;
2445           }
2446         }
2447         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2448       }
2449       /* Build Topology */
2450       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2451       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2452       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2453       /* Cells */
2454       if (rank == 0) {
2455         for (i = 0, c = 0; i < numVerts; ++i) {
2456           for (j = 0; j < i; ++j) {
2457             for (k = 0; k < j; ++k) {
2458               for (l = 0; l < k; ++l) {
2459                 if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i] && graph[l * numVerts + i] && graph[l * numVerts + j] && graph[l * numVerts + k]) {
2460                   cone[0] = firstVertex + i;
2461                   cone[1] = firstVertex + j;
2462                   cone[2] = firstVertex + k;
2463                   cone[3] = firstVertex + l;
2464                   /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
2465                   {
2466                     const PetscInt epsilon[4][4][4][4] = {
2467                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, -1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 0, 0}, {0, 1, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 1, 0}, {0, -1, 0, 0}, {0, 0, 0, 0}}},
2468 
2469                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}}, {{0, 0, -1, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}}},
2470 
2471                       {{{0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, 0, 0}, {0, -1, 0, 0}}, {{0, 0, 0, -1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 1, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}},
2472 
2473                       {{{0, 0, 0, 0}, {0, 0, -1, 0}, {0, 1, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 1, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}}, {{0, -1, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}} }
2474                     };
2475                     PetscReal normal[4];
2476                     PetscInt  e, f, g;
2477 
2478                     for (d = 0; d < embedDim; ++d) {
2479                       normal[d] = 0.0;
2480                       for (e = 0; e < embedDim; ++e) {
2481                         for (f = 0; f < embedDim; ++f) {
2482                           for (g = 0; g < embedDim; ++g) {
2483                             normal[d] += epsilon[d][e][f][g] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]) * (coordsIn[l * embedDim + f] - coordsIn[i * embedDim + f]);
2484                           }
2485                         }
2486                       }
2487                     }
2488                     if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2489                       PetscInt tmp = cone[1];
2490                       cone[1]      = cone[2];
2491                       cone[2]      = tmp;
2492                     }
2493                   }
2494                   PetscCall(DMPlexSetCone(dm, c++, cone));
2495                 }
2496               }
2497             }
2498           }
2499         }
2500       }
2501       PetscCall(DMPlexSymmetrize(dm));
2502       PetscCall(DMPlexStratify(dm));
2503       PetscCall(PetscFree(graph));
2504     }
2505     break;
2506   default:
2507     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unsupported dimension for sphere: %" PetscInt_FMT, dim);
2508   }
2509   /* Create coordinates */
2510   PetscCall(DMGetCoordinateSection(dm, &coordSection));
2511   PetscCall(PetscSectionSetNumFields(coordSection, 1));
2512   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, embedDim));
2513   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVerts));
2514   for (v = firstVertex; v < firstVertex + numVerts; ++v) {
2515     PetscCall(PetscSectionSetDof(coordSection, v, embedDim));
2516     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, embedDim));
2517   }
2518   PetscCall(PetscSectionSetUp(coordSection));
2519   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2520   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2521   PetscCall(VecSetBlockSize(coordinates, embedDim));
2522   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2523   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2524   PetscCall(VecSetType(coordinates, VECSTANDARD));
2525   PetscCall(VecGetArray(coordinates, &coords));
2526   for (v = 0; v < numVerts; ++v)
2527     for (d = 0; d < embedDim; ++d) coords[v * embedDim + d] = coordsIn[v * embedDim + d];
2528   PetscCall(VecRestoreArray(coordinates, &coords));
2529   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2530   PetscCall(VecDestroy(&coordinates));
2531   PetscCall(PetscFree(coordsIn));
2532   {
2533     DM          cdm;
2534     PetscDS     cds;
2535     PetscScalar c = R;
2536 
2537     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToSphere));
2538     PetscCall(DMGetCoordinateDM(dm, &cdm));
2539     PetscCall(DMGetDS(cdm, &cds));
2540     PetscCall(PetscDSSetConstants(cds, 1, &c));
2541   }
2542   /* Wait for coordinate creation before doing in-place modification */
2543   if (simplex) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2544   PetscFunctionReturn(0);
2545 }
2546 
2547 typedef void (*TPSEvaluateFunc)(const PetscReal[], PetscReal *, PetscReal[], PetscReal (*)[3]);
2548 
2549 /*
2550  The Schwarz P implicit surface is
2551 
2552      f(x) = cos(x0) + cos(x1) + cos(x2) = 0
2553 */
2554 static void TPSEvaluate_SchwarzP(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2555 {
2556   PetscReal c[3] = {PetscCosReal(y[0] * PETSC_PI), PetscCosReal(y[1] * PETSC_PI), PetscCosReal(y[2] * PETSC_PI)};
2557   PetscReal g[3] = {-PetscSinReal(y[0] * PETSC_PI), -PetscSinReal(y[1] * PETSC_PI), -PetscSinReal(y[2] * PETSC_PI)};
2558   f[0]           = c[0] + c[1] + c[2];
2559   for (PetscInt i = 0; i < 3; i++) {
2560     grad[i] = PETSC_PI * g[i];
2561     for (PetscInt j = 0; j < 3; j++) hess[i][j] = (i == j) ? -PetscSqr(PETSC_PI) * c[i] : 0.;
2562   }
2563 }
2564 
2565 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2566 static PetscErrorCode TPSExtrudeNormalFunc_SchwarzP(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2567 {
2568   for (PetscInt i = 0; i < 3; i++) u[i] = -PETSC_PI * PetscSinReal(x[i] * PETSC_PI);
2569   return 0;
2570 }
2571 
2572 /*
2573  The Gyroid implicit surface is
2574 
2575  f(x,y,z) = sin(pi * x) * cos (pi * (y + 1/2))  + sin(pi * (y + 1/2)) * cos(pi * (z + 1/4)) + sin(pi * (z + 1/4)) * cos(pi * x)
2576 
2577 */
2578 static void TPSEvaluate_Gyroid(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2579 {
2580   PetscReal s[3] = {PetscSinReal(PETSC_PI * y[0]), PetscSinReal(PETSC_PI * (y[1] + .5)), PetscSinReal(PETSC_PI * (y[2] + .25))};
2581   PetscReal c[3] = {PetscCosReal(PETSC_PI * y[0]), PetscCosReal(PETSC_PI * (y[1] + .5)), PetscCosReal(PETSC_PI * (y[2] + .25))};
2582   f[0]           = s[0] * c[1] + s[1] * c[2] + s[2] * c[0];
2583   grad[0]        = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2584   grad[1]        = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2585   grad[2]        = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2586   hess[0][0]     = -PetscSqr(PETSC_PI) * (s[0] * c[1] + s[2] * c[0]);
2587   hess[0][1]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2588   hess[0][2]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2589   hess[1][0]     = -PetscSqr(PETSC_PI) * (s[1] * c[2] + s[0] * c[1]);
2590   hess[1][1]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2591   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2592   hess[2][0]     = -PetscSqr(PETSC_PI) * (s[2] * c[0] + s[1] * c[2]);
2593   hess[2][1]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2594   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2595 }
2596 
2597 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2598 static PetscErrorCode TPSExtrudeNormalFunc_Gyroid(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2599 {
2600   PetscReal s[3] = {PetscSinReal(PETSC_PI * x[0]), PetscSinReal(PETSC_PI * (x[1] + .5)), PetscSinReal(PETSC_PI * (x[2] + .25))};
2601   PetscReal c[3] = {PetscCosReal(PETSC_PI * x[0]), PetscCosReal(PETSC_PI * (x[1] + .5)), PetscCosReal(PETSC_PI * (x[2] + .25))};
2602   u[0]           = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2603   u[1]           = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2604   u[2]           = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2605   return 0;
2606 }
2607 
2608 /*
2609    We wish to solve
2610 
2611          min_y || y - x ||^2  subject to f(y) = 0
2612 
2613    Let g(y) = grad(f).  The minimization problem is equivalent to asking to satisfy
2614    f(y) = 0 and (y-x) is parallel to g(y).  We do this by using Householder QR to obtain a basis for the
2615    tangent space and ask for both components in the tangent space to be zero.
2616 
2617    Take g to be a column vector and compute the "full QR" factorization Q R = g,
2618    where Q = I - 2 n n^T is a symmetric orthogonal matrix.
2619    The first column of Q is parallel to g so the remaining two columns span the null space.
2620    Let Qn = Q[:,1:] be those remaining columns.  Then Qn Qn^T is an orthogonal projector into the tangent space.
2621    Since Q is symmetric, this is equivalent to multipyling by Q and taking the last two entries.
2622    In total, we have a system of 3 equations in 3 unknowns:
2623 
2624      f(y) = 0                       1 equation
2625      Qn^T (y - x) = 0               2 equations
2626 
2627    Here, we compute the residual and Jacobian of this system.
2628 */
2629 static void TPSNearestPointResJac(TPSEvaluateFunc feval, const PetscScalar x[], const PetscScalar y[], PetscScalar res[], PetscScalar J[])
2630 {
2631   PetscReal yreal[3] = {PetscRealPart(y[0]), PetscRealPart(y[1]), PetscRealPart(y[2])};
2632   PetscReal d[3]     = {PetscRealPart(y[0] - x[0]), PetscRealPart(y[1] - x[1]), PetscRealPart(y[2] - x[2])};
2633   PetscReal f, grad[3], n[3], norm, norm_y[3], nd, nd_y[3], sign;
2634   PetscReal n_y[3][3] = {
2635     {0, 0, 0},
2636     {0, 0, 0},
2637     {0, 0, 0}
2638   };
2639 
2640   feval(yreal, &f, grad, n_y);
2641 
2642   for (PetscInt i = 0; i < 3; i++) n[i] = grad[i];
2643   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2644   for (PetscInt i = 0; i < 3; i++) norm_y[i] = 1. / norm * n[i] * n_y[i][i];
2645 
2646   // Define the Householder reflector
2647   sign = n[0] >= 0 ? 1. : -1.;
2648   n[0] += norm * sign;
2649   for (PetscInt i = 0; i < 3; i++) n_y[0][i] += norm_y[i] * sign;
2650 
2651   norm      = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2652   norm_y[0] = 1. / norm * (n[0] * n_y[0][0]);
2653   norm_y[1] = 1. / norm * (n[0] * n_y[0][1] + n[1] * n_y[1][1]);
2654   norm_y[2] = 1. / norm * (n[0] * n_y[0][2] + n[2] * n_y[2][2]);
2655 
2656   for (PetscInt i = 0; i < 3; i++) {
2657     n[i] /= norm;
2658     for (PetscInt j = 0; j < 3; j++) {
2659       // note that n[i] is n_old[i]/norm when executing the code below
2660       n_y[i][j] = n_y[i][j] / norm - n[i] / norm * norm_y[j];
2661     }
2662   }
2663 
2664   nd = n[0] * d[0] + n[1] * d[1] + n[2] * d[2];
2665   for (PetscInt i = 0; i < 3; i++) nd_y[i] = n[i] + n_y[0][i] * d[0] + n_y[1][i] * d[1] + n_y[2][i] * d[2];
2666 
2667   res[0] = f;
2668   res[1] = d[1] - 2 * n[1] * nd;
2669   res[2] = d[2] - 2 * n[2] * nd;
2670   // J[j][i] is J_{ij} (column major)
2671   for (PetscInt j = 0; j < 3; j++) {
2672     J[0 + j * 3] = grad[j];
2673     J[1 + j * 3] = (j == 1) * 1. - 2 * (n_y[1][j] * nd + n[1] * nd_y[j]);
2674     J[2 + j * 3] = (j == 2) * 1. - 2 * (n_y[2][j] * nd + n[2] * nd_y[j]);
2675   }
2676 }
2677 
2678 /*
2679    Project x to the nearest point on the implicit surface using Newton's method.
2680 */
2681 static PetscErrorCode TPSNearestPoint(TPSEvaluateFunc feval, PetscScalar x[])
2682 {
2683   PetscScalar y[3] = {x[0], x[1], x[2]}; // Initial guess
2684 
2685   PetscFunctionBegin;
2686   for (PetscInt iter = 0; iter < 10; iter++) {
2687     PetscScalar res[3], J[9];
2688     PetscReal   resnorm;
2689     TPSNearestPointResJac(feval, x, y, res, J);
2690     resnorm = PetscSqrtReal(PetscSqr(PetscRealPart(res[0])) + PetscSqr(PetscRealPart(res[1])) + PetscSqr(PetscRealPart(res[2])));
2691     if (0) { // Turn on this monitor if you need to confirm quadratic convergence
2692       PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%" PetscInt_FMT "] res [%g %g %g]\n", iter, (double)PetscRealPart(res[0]), (double)PetscRealPart(res[1]), (double)PetscRealPart(res[2])));
2693     }
2694     if (resnorm < PETSC_SMALL) break;
2695 
2696     // Take the Newton step
2697     PetscCall(PetscKernel_A_gets_inverse_A_3(J, 0., PETSC_FALSE, NULL));
2698     PetscKernel_v_gets_v_minus_A_times_w_3(y, J, res);
2699   }
2700   for (PetscInt i = 0; i < 3; i++) x[i] = y[i];
2701   PetscFunctionReturn(0);
2702 }
2703 
2704 const char *const DMPlexTPSTypes[] = {"SCHWARZ_P", "GYROID", "DMPlexTPSType", "DMPLEX_TPS_", NULL};
2705 
2706 static PetscErrorCode DMPlexCreateTPSMesh_Internal(DM dm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness)
2707 {
2708   PetscMPIInt rank;
2709   PetscInt    topoDim = 2, spaceDim = 3, numFaces = 0, numVertices = 0, numEdges = 0;
2710   PetscInt(*edges)[2] = NULL, *edgeSets = NULL;
2711   PetscInt            *cells_flat = NULL;
2712   PetscReal           *vtxCoords  = NULL;
2713   TPSEvaluateFunc      evalFunc   = NULL;
2714   PetscSimplePointFunc normalFunc = NULL;
2715   DMLabel              label;
2716 
2717   PetscFunctionBegin;
2718   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2719   PetscCheck((layers != 0) ^ (thickness == 0.), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Layers %" PetscInt_FMT " must be nonzero iff thickness %g is nonzero", layers, (double)thickness);
2720   switch (tpstype) {
2721   case DMPLEX_TPS_SCHWARZ_P:
2722     PetscCheck(!periodic || (periodic[0] == DM_BOUNDARY_NONE && periodic[1] == DM_BOUNDARY_NONE && periodic[2] == DM_BOUNDARY_NONE), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Schwarz P does not support periodic meshes");
2723     if (rank == 0) {
2724       PetscInt(*cells)[6][4][4] = NULL; // [junction, junction-face, cell, conn]
2725       PetscInt  Njunctions = 0, Ncuts = 0, Npipes[3], vcount;
2726       PetscReal L = 1;
2727 
2728       Npipes[0]   = (extent[0] + 1) * extent[1] * extent[2];
2729       Npipes[1]   = extent[0] * (extent[1] + 1) * extent[2];
2730       Npipes[2]   = extent[0] * extent[1] * (extent[2] + 1);
2731       Njunctions  = extent[0] * extent[1] * extent[2];
2732       Ncuts       = 2 * (extent[0] * extent[1] + extent[1] * extent[2] + extent[2] * extent[0]);
2733       numVertices = 4 * (Npipes[0] + Npipes[1] + Npipes[2]) + 8 * Njunctions;
2734       PetscCall(PetscMalloc1(3 * numVertices, &vtxCoords));
2735       PetscCall(PetscMalloc1(Njunctions, &cells));
2736       PetscCall(PetscMalloc1(Ncuts * 4, &edges));
2737       PetscCall(PetscMalloc1(Ncuts * 4, &edgeSets));
2738       // x-normal pipes
2739       vcount = 0;
2740       for (PetscInt i = 0; i < extent[0] + 1; i++) {
2741         for (PetscInt j = 0; j < extent[1]; j++) {
2742           for (PetscInt k = 0; k < extent[2]; k++) {
2743             for (PetscInt l = 0; l < 4; l++) {
2744               vtxCoords[vcount++] = (2 * i - 1) * L;
2745               vtxCoords[vcount++] = 2 * j * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2746               vtxCoords[vcount++] = 2 * k * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2747             }
2748           }
2749         }
2750       }
2751       // y-normal pipes
2752       for (PetscInt i = 0; i < extent[0]; i++) {
2753         for (PetscInt j = 0; j < extent[1] + 1; j++) {
2754           for (PetscInt k = 0; k < extent[2]; k++) {
2755             for (PetscInt l = 0; l < 4; l++) {
2756               vtxCoords[vcount++] = 2 * i * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2757               vtxCoords[vcount++] = (2 * j - 1) * L;
2758               vtxCoords[vcount++] = 2 * k * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2759             }
2760           }
2761         }
2762       }
2763       // z-normal pipes
2764       for (PetscInt i = 0; i < extent[0]; i++) {
2765         for (PetscInt j = 0; j < extent[1]; j++) {
2766           for (PetscInt k = 0; k < extent[2] + 1; k++) {
2767             for (PetscInt l = 0; l < 4; l++) {
2768               vtxCoords[vcount++] = 2 * i * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2769               vtxCoords[vcount++] = 2 * j * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2770               vtxCoords[vcount++] = (2 * k - 1) * L;
2771             }
2772           }
2773         }
2774       }
2775       // junctions
2776       for (PetscInt i = 0; i < extent[0]; i++) {
2777         for (PetscInt j = 0; j < extent[1]; j++) {
2778           for (PetscInt k = 0; k < extent[2]; k++) {
2779             const PetscInt J = (i * extent[1] + j) * extent[2] + k, Jvoff = (Npipes[0] + Npipes[1] + Npipes[2]) * 4 + J * 8;
2780             PetscCheck(vcount / 3 == Jvoff, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected vertex count");
2781             for (PetscInt ii = 0; ii < 2; ii++) {
2782               for (PetscInt jj = 0; jj < 2; jj++) {
2783                 for (PetscInt kk = 0; kk < 2; kk++) {
2784                   double Ls           = (1 - sqrt(2) / 4) * L;
2785                   vtxCoords[vcount++] = 2 * i * L + (2 * ii - 1) * Ls;
2786                   vtxCoords[vcount++] = 2 * j * L + (2 * jj - 1) * Ls;
2787                   vtxCoords[vcount++] = 2 * k * L + (2 * kk - 1) * Ls;
2788                 }
2789               }
2790             }
2791             const PetscInt jfaces[3][2][4] = {
2792               {{3, 1, 0, 2}, {7, 5, 4, 6}}, // x-aligned
2793               {{5, 4, 0, 1}, {7, 6, 2, 3}}, // y-aligned
2794               {{6, 2, 0, 4}, {7, 3, 1, 5}}  // z-aligned
2795             };
2796             const PetscInt pipe_lo[3] = {// vertex numbers of pipes
2797                                          ((i * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + Npipes[0] + Npipes[1]) * 4};
2798             const PetscInt pipe_hi[3] = {// vertex numbers of pipes
2799                                          (((i + 1) * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j + 1) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + 1 + Npipes[0] + Npipes[1]) * 4};
2800             for (PetscInt dir = 0; dir < 3; dir++) { // x,y,z
2801               const PetscInt ijk[3] = {i, j, k};
2802               for (PetscInt l = 0; l < 4; l++) { // rotations
2803                 cells[J][dir * 2 + 0][l][0] = pipe_lo[dir] + l;
2804                 cells[J][dir * 2 + 0][l][1] = Jvoff + jfaces[dir][0][l];
2805                 cells[J][dir * 2 + 0][l][2] = Jvoff + jfaces[dir][0][(l - 1 + 4) % 4];
2806                 cells[J][dir * 2 + 0][l][3] = pipe_lo[dir] + (l - 1 + 4) % 4;
2807                 cells[J][dir * 2 + 1][l][0] = Jvoff + jfaces[dir][1][l];
2808                 cells[J][dir * 2 + 1][l][1] = pipe_hi[dir] + l;
2809                 cells[J][dir * 2 + 1][l][2] = pipe_hi[dir] + (l - 1 + 4) % 4;
2810                 cells[J][dir * 2 + 1][l][3] = Jvoff + jfaces[dir][1][(l - 1 + 4) % 4];
2811                 if (ijk[dir] == 0) {
2812                   edges[numEdges][0] = pipe_lo[dir] + l;
2813                   edges[numEdges][1] = pipe_lo[dir] + (l + 1) % 4;
2814                   edgeSets[numEdges] = dir * 2 + 1;
2815                   numEdges++;
2816                 }
2817                 if (ijk[dir] + 1 == extent[dir]) {
2818                   edges[numEdges][0] = pipe_hi[dir] + l;
2819                   edges[numEdges][1] = pipe_hi[dir] + (l + 1) % 4;
2820                   edgeSets[numEdges] = dir * 2 + 2;
2821                   numEdges++;
2822                 }
2823               }
2824             }
2825           }
2826         }
2827       }
2828       PetscCheck(numEdges == Ncuts * 4, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Edge count %" PetscInt_FMT " incompatible with number of cuts %" PetscInt_FMT, numEdges, Ncuts);
2829       numFaces   = 24 * Njunctions;
2830       cells_flat = cells[0][0][0];
2831     }
2832     evalFunc   = TPSEvaluate_SchwarzP;
2833     normalFunc = TPSExtrudeNormalFunc_SchwarzP;
2834     break;
2835   case DMPLEX_TPS_GYROID:
2836     if (rank == 0) {
2837       // This is a coarse mesh approximation of the gyroid shifted to being the zero of the level set
2838       //
2839       //     sin(pi*x)*cos(pi*(y+1/2)) + sin(pi*(y+1/2))*cos(pi*(z+1/4)) + sin(pi*(z+1/4))*cos(x)
2840       //
2841       // on the cell [0,2]^3.
2842       //
2843       // Think about dividing that cell into four columns, and focus on the column [0,1]x[0,1]x[0,2].
2844       // If you looked at the gyroid in that column at different slices of z you would see that it kind of spins
2845       // like a boomerang:
2846       //
2847       //     z = 0          z = 1/4        z = 1/2        z = 3/4     //
2848       //     -----          -------        -------        -------     //
2849       //                                                              //
2850       //     +       +      +       +      +       +      +   \   +   //
2851       //      \                                   /            \      //
2852       //       \            `-_   _-'            /              }     //
2853       //        *-_            `-'            _-'              /      //
2854       //     +     `-+      +       +      +-'     +      +   /   +   //
2855       //                                                              //
2856       //                                                              //
2857       //     z = 1          z = 5/4        z = 3/2        z = 7/4     //
2858       //     -----          -------        -------        -------     //
2859       //                                                              //
2860       //     +-_     +      +       +      +     _-+      +   /   +   //
2861       //        `-_            _-_            _-`            /        //
2862       //           \        _-'   `-_        /              {         //
2863       //            \                       /                \        //
2864       //     +       +      +       +      +       +      +   \   +   //
2865       //
2866       //
2867       // This course mesh approximates each of these slices by two line segments,
2868       // and then connects the segments in consecutive layers with quadrilateral faces.
2869       // All of the end points of the segments are multiples of 1/4 except for the
2870       // point * in the picture for z = 0 above and the similar points in other layers.
2871       // That point is at (gamma, gamma, 0), where gamma is calculated below.
2872       //
2873       // The column  [1,2]x[1,2]x[0,2] looks the same as this column;
2874       // The columns [1,2]x[0,1]x[0,2] and [0,1]x[1,2]x[0,2] are mirror images.
2875       //
2876       // As for how this method turned into the names given to the vertices:
2877       // that was not systematic, it was just the way it worked out in my handwritten notes.
2878 
2879       PetscInt facesPerBlock = 64;
2880       PetscInt vertsPerBlock = 56;
2881       PetscInt extentPlus[3];
2882       PetscInt numBlocks, numBlocksPlus;
2883       const PetscInt A = 0, B = 1, C = 2, D = 3, E = 4, F = 5, G = 6, H = 7, II = 8, J = 9, K = 10, L = 11, M = 12, N = 13, O = 14, P = 15, Q = 16, R = 17, S = 18, T = 19, U = 20, V = 21, W = 22, X = 23, Y = 24, Z = 25, Ap = 26, Bp = 27, Cp = 28, Dp = 29, Ep = 30, Fp = 31, Gp = 32, Hp = 33, Ip = 34, Jp = 35, Kp = 36, Lp = 37, Mp = 38, Np = 39, Op = 40, Pp = 41, Qp = 42, Rp = 43, Sp = 44, Tp = 45, Up = 46, Vp = 47, Wp = 48, Xp = 49, Yp = 50, Zp = 51, Aq = 52, Bq = 53, Cq = 54, Dq = 55;
2884       const PetscInt pattern[64][4] = {
2885   /* face to vertex within the coarse discretization of a single gyroid block */
2886   /* layer 0 */
2887         {A,           C,           K,           G          },
2888         {C,           B,           II,          K          },
2889         {D,           A,           H,           L          },
2890         {B + 56 * 1,  D,           L,           J          },
2891         {E,           B + 56 * 1,  J,           N          },
2892         {A + 56 * 2,  E,           N,           H + 56 * 2 },
2893         {F,           A + 56 * 2,  G + 56 * 2,  M          },
2894         {B,           F,           M,           II         },
2895  /* layer 1 */
2896         {G,           K,           Q,           O          },
2897         {K,           II,          P,           Q          },
2898         {L,           H,           O + 56 * 1,  R          },
2899         {J,           L,           R,           P          },
2900         {N,           J,           P,           S          },
2901         {H + 56 * 2,  N,           S,           O + 56 * 3 },
2902         {M,           G + 56 * 2,  O + 56 * 2,  T          },
2903         {II,          M,           T,           P          },
2904  /* layer 2 */
2905         {O,           Q,           Y,           U          },
2906         {Q,           P,           W,           Y          },
2907         {R,           O + 56 * 1,  U + 56 * 1,  Ap         },
2908         {P,           R,           Ap,          W          },
2909         {S,           P,           X,           Bp         },
2910         {O + 56 * 3,  S,           Bp,          V + 56 * 1 },
2911         {T,           O + 56 * 2,  V,           Z          },
2912         {P,           T,           Z,           X          },
2913  /* layer 3 */
2914         {U,           Y,           Ep,          Dp         },
2915         {Y,           W,           Cp,          Ep         },
2916         {Ap,          U + 56 * 1,  Dp + 56 * 1, Gp         },
2917         {W,           Ap,          Gp,          Cp         },
2918         {Bp,          X,           Cp + 56 * 2, Fp         },
2919         {V + 56 * 1,  Bp,          Fp,          Dp + 56 * 1},
2920         {Z,           V,           Dp,          Hp         },
2921         {X,           Z,           Hp,          Cp + 56 * 2},
2922  /* layer 4 */
2923         {Dp,          Ep,          Mp,          Kp         },
2924         {Ep,          Cp,          Ip,          Mp         },
2925         {Gp,          Dp + 56 * 1, Lp,          Np         },
2926         {Cp,          Gp,          Np,          Jp         },
2927         {Fp,          Cp + 56 * 2, Jp + 56 * 2, Pp         },
2928         {Dp + 56 * 1, Fp,          Pp,          Lp         },
2929         {Hp,          Dp,          Kp,          Op         },
2930         {Cp + 56 * 2, Hp,          Op,          Ip + 56 * 2},
2931  /* layer 5 */
2932         {Kp,          Mp,          Sp,          Rp         },
2933         {Mp,          Ip,          Qp,          Sp         },
2934         {Np,          Lp,          Rp,          Tp         },
2935         {Jp,          Np,          Tp,          Qp + 56 * 1},
2936         {Pp,          Jp + 56 * 2, Qp + 56 * 3, Up         },
2937         {Lp,          Pp,          Up,          Rp         },
2938         {Op,          Kp,          Rp,          Vp         },
2939         {Ip + 56 * 2, Op,          Vp,          Qp + 56 * 2},
2940  /* layer 6 */
2941         {Rp,          Sp,          Aq,          Yp         },
2942         {Sp,          Qp,          Wp,          Aq         },
2943         {Tp,          Rp,          Yp,          Cq         },
2944         {Qp + 56 * 1, Tp,          Cq,          Wp + 56 * 1},
2945         {Up,          Qp + 56 * 3, Xp + 56 * 1, Dq         },
2946         {Rp,          Up,          Dq,          Zp         },
2947         {Vp,          Rp,          Zp,          Bq         },
2948         {Qp + 56 * 2, Vp,          Bq,          Xp         },
2949  /* layer 7 (the top is the periodic image of the bottom of layer 0) */
2950         {Yp,          Aq,          C + 56 * 4,  A + 56 * 4 },
2951         {Aq,          Wp,          B + 56 * 4,  C + 56 * 4 },
2952         {Cq,          Yp,          A + 56 * 4,  D + 56 * 4 },
2953         {Wp + 56 * 1, Cq,          D + 56 * 4,  B + 56 * 5 },
2954         {Dq,          Xp + 56 * 1, B + 56 * 5,  E + 56 * 4 },
2955         {Zp,          Dq,          E + 56 * 4,  A + 56 * 6 },
2956         {Bq,          Zp,          A + 56 * 6,  F + 56 * 4 },
2957         {Xp,          Bq,          F + 56 * 4,  B + 56 * 4 }
2958       };
2959       const PetscReal gamma                = PetscAcosReal((PetscSqrtReal(3.) - 1.) / PetscSqrtReal(2.)) / PETSC_PI;
2960       const PetscReal patternCoords[56][3] = {
2961         {1.,        0.,        0.  }, /* A  */
2962         {0.,        1.,        0.  }, /* B  */
2963         {gamma,     gamma,     0.  }, /* C  */
2964         {1 + gamma, 1 - gamma, 0.  }, /* D  */
2965         {2 - gamma, 2 - gamma, 0.  }, /* E  */
2966         {1 - gamma, 1 + gamma, 0.  }, /* F  */
2967 
2968         {.5,        0,         .25 }, /* G  */
2969         {1.5,       0.,        .25 }, /* H  */
2970         {.5,        1.,        .25 }, /* II */
2971         {1.5,       1.,        .25 }, /* J  */
2972         {.25,       .5,        .25 }, /* K  */
2973         {1.25,      .5,        .25 }, /* L  */
2974         {.75,       1.5,       .25 }, /* M  */
2975         {1.75,      1.5,       .25 }, /* N  */
2976 
2977         {0.,        0.,        .5  }, /* O  */
2978         {1.,        1.,        .5  }, /* P  */
2979         {gamma,     1 - gamma, .5  }, /* Q  */
2980         {1 + gamma, gamma,     .5  }, /* R  */
2981         {2 - gamma, 1 + gamma, .5  }, /* S  */
2982         {1 - gamma, 2 - gamma, .5  }, /* T  */
2983 
2984         {0.,        .5,        .75 }, /* U  */
2985         {0.,        1.5,       .75 }, /* V  */
2986         {1.,        .5,        .75 }, /* W  */
2987         {1.,        1.5,       .75 }, /* X  */
2988         {.5,        .75,       .75 }, /* Y  */
2989         {.5,        1.75,      .75 }, /* Z  */
2990         {1.5,       .25,       .75 }, /* Ap */
2991         {1.5,       1.25,      .75 }, /* Bp */
2992 
2993         {1.,        0.,        1.  }, /* Cp */
2994         {0.,        1.,        1.  }, /* Dp */
2995         {1 - gamma, 1 - gamma, 1.  }, /* Ep */
2996         {1 + gamma, 1 + gamma, 1.  }, /* Fp */
2997         {2 - gamma, gamma,     1.  }, /* Gp */
2998         {gamma,     2 - gamma, 1.  }, /* Hp */
2999 
3000         {.5,        0.,        1.25}, /* Ip */
3001         {1.5,       0.,        1.25}, /* Jp */
3002         {.5,        1.,        1.25}, /* Kp */
3003         {1.5,       1.,        1.25}, /* Lp */
3004         {.75,       .5,        1.25}, /* Mp */
3005         {1.75,      .5,        1.25}, /* Np */
3006         {.25,       1.5,       1.25}, /* Op */
3007         {1.25,      1.5,       1.25}, /* Pp */
3008 
3009         {0.,        0.,        1.5 }, /* Qp */
3010         {1.,        1.,        1.5 }, /* Rp */
3011         {1 - gamma, gamma,     1.5 }, /* Sp */
3012         {2 - gamma, 1 - gamma, 1.5 }, /* Tp */
3013         {1 + gamma, 2 - gamma, 1.5 }, /* Up */
3014         {gamma,     1 + gamma, 1.5 }, /* Vp */
3015 
3016         {0.,        .5,        1.75}, /* Wp */
3017         {0.,        1.5,       1.75}, /* Xp */
3018         {1.,        .5,        1.75}, /* Yp */
3019         {1.,        1.5,       1.75}, /* Zp */
3020         {.5,        .25,       1.75}, /* Aq */
3021         {.5,        1.25,      1.75}, /* Bq */
3022         {1.5,       .75,       1.75}, /* Cq */
3023         {1.5,       1.75,      1.75}, /* Dq */
3024       };
3025       PetscInt(*cells)[64][4] = NULL;
3026       PetscBool *seen;
3027       PetscInt  *vertToTrueVert;
3028       PetscInt   count;
3029 
3030       for (PetscInt i = 0; i < 3; i++) extentPlus[i] = extent[i] + 1;
3031       numBlocks = 1;
3032       for (PetscInt i = 0; i < 3; i++) numBlocks *= extent[i];
3033       numBlocksPlus = 1;
3034       for (PetscInt i = 0; i < 3; i++) numBlocksPlus *= extentPlus[i];
3035       numFaces = numBlocks * facesPerBlock;
3036       PetscCall(PetscMalloc1(numBlocks, &cells));
3037       PetscCall(PetscCalloc1(numBlocksPlus * vertsPerBlock, &seen));
3038       for (PetscInt k = 0; k < extent[2]; k++) {
3039         for (PetscInt j = 0; j < extent[1]; j++) {
3040           for (PetscInt i = 0; i < extent[0]; i++) {
3041             for (PetscInt f = 0; f < facesPerBlock; f++) {
3042               for (PetscInt v = 0; v < 4; v++) {
3043                 PetscInt vertRaw     = pattern[f][v];
3044                 PetscInt blockidx    = vertRaw / 56;
3045                 PetscInt patternvert = vertRaw % 56;
3046                 PetscInt xplus       = (blockidx & 1);
3047                 PetscInt yplus       = (blockidx & 2) >> 1;
3048                 PetscInt zplus       = (blockidx & 4) >> 2;
3049                 PetscInt zcoord      = (periodic && periodic[2] == DM_BOUNDARY_PERIODIC) ? ((k + zplus) % extent[2]) : (k + zplus);
3050                 PetscInt ycoord      = (periodic && periodic[1] == DM_BOUNDARY_PERIODIC) ? ((j + yplus) % extent[1]) : (j + yplus);
3051                 PetscInt xcoord      = (periodic && periodic[0] == DM_BOUNDARY_PERIODIC) ? ((i + xplus) % extent[0]) : (i + xplus);
3052                 PetscInt vert        = ((zcoord * extentPlus[1] + ycoord) * extentPlus[0] + xcoord) * 56 + patternvert;
3053 
3054                 cells[(k * extent[1] + j) * extent[0] + i][f][v] = vert;
3055                 seen[vert]                                       = PETSC_TRUE;
3056               }
3057             }
3058           }
3059         }
3060       }
3061       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++)
3062         if (seen[i]) numVertices++;
3063       count = 0;
3064       PetscCall(PetscMalloc1(numBlocksPlus * vertsPerBlock, &vertToTrueVert));
3065       PetscCall(PetscMalloc1(numVertices * 3, &vtxCoords));
3066       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) vertToTrueVert[i] = -1;
3067       for (PetscInt k = 0; k < extentPlus[2]; k++) {
3068         for (PetscInt j = 0; j < extentPlus[1]; j++) {
3069           for (PetscInt i = 0; i < extentPlus[0]; i++) {
3070             for (PetscInt v = 0; v < vertsPerBlock; v++) {
3071               PetscInt vIdx = ((k * extentPlus[1] + j) * extentPlus[0] + i) * vertsPerBlock + v;
3072 
3073               if (seen[vIdx]) {
3074                 PetscInt thisVert;
3075 
3076                 vertToTrueVert[vIdx] = thisVert = count++;
3077 
3078                 for (PetscInt d = 0; d < 3; d++) vtxCoords[3 * thisVert + d] = patternCoords[v][d];
3079                 vtxCoords[3 * thisVert + 0] += i * 2;
3080                 vtxCoords[3 * thisVert + 1] += j * 2;
3081                 vtxCoords[3 * thisVert + 2] += k * 2;
3082               }
3083             }
3084           }
3085         }
3086       }
3087       for (PetscInt i = 0; i < numBlocks; i++) {
3088         for (PetscInt f = 0; f < facesPerBlock; f++) {
3089           for (PetscInt v = 0; v < 4; v++) cells[i][f][v] = vertToTrueVert[cells[i][f][v]];
3090         }
3091       }
3092       PetscCall(PetscFree(vertToTrueVert));
3093       PetscCall(PetscFree(seen));
3094       cells_flat = cells[0][0];
3095       numEdges   = 0;
3096       for (PetscInt i = 0; i < numFaces; i++) {
3097         for (PetscInt e = 0; e < 4; e++) {
3098           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3099           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3100 
3101           for (PetscInt d = 0; d < 3; d++) {
3102             if (!periodic || periodic[0] != DM_BOUNDARY_PERIODIC) {
3103               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) numEdges++;
3104               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) numEdges++;
3105             }
3106           }
3107         }
3108       }
3109       PetscCall(PetscMalloc1(numEdges, &edges));
3110       PetscCall(PetscMalloc1(numEdges, &edgeSets));
3111       for (PetscInt edge = 0, i = 0; i < numFaces; i++) {
3112         for (PetscInt e = 0; e < 4; e++) {
3113           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3114           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3115 
3116           for (PetscInt d = 0; d < 3; d++) {
3117             if (!periodic || periodic[d] != DM_BOUNDARY_PERIODIC) {
3118               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) {
3119                 edges[edge][0]   = ev[0];
3120                 edges[edge][1]   = ev[1];
3121                 edgeSets[edge++] = 2 * d;
3122               }
3123               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) {
3124                 edges[edge][0]   = ev[0];
3125                 edges[edge][1]   = ev[1];
3126                 edgeSets[edge++] = 2 * d + 1;
3127               }
3128             }
3129           }
3130         }
3131       }
3132     }
3133     evalFunc   = TPSEvaluate_Gyroid;
3134     normalFunc = TPSExtrudeNormalFunc_Gyroid;
3135     break;
3136   }
3137 
3138   PetscCall(DMSetDimension(dm, topoDim));
3139   if (rank == 0) PetscCall(DMPlexBuildFromCellList(dm, numFaces, numVertices, 4, cells_flat));
3140   else PetscCall(DMPlexBuildFromCellList(dm, 0, 0, 0, NULL));
3141   PetscCall(PetscFree(cells_flat));
3142   {
3143     DM idm;
3144     PetscCall(DMPlexInterpolate(dm, &idm));
3145     PetscCall(DMPlexReplace_Internal(dm, &idm));
3146   }
3147   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, vtxCoords));
3148   else PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, NULL));
3149   PetscCall(PetscFree(vtxCoords));
3150 
3151   PetscCall(DMCreateLabel(dm, "Face Sets"));
3152   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3153   for (PetscInt e = 0; e < numEdges; e++) {
3154     PetscInt        njoin;
3155     const PetscInt *join, verts[] = {numFaces + edges[e][0], numFaces + edges[e][1]};
3156     PetscCall(DMPlexGetJoin(dm, 2, verts, &njoin, &join));
3157     PetscCheck(njoin == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected unique join of vertices %" PetscInt_FMT " and %" PetscInt_FMT, edges[e][0], edges[e][1]);
3158     PetscCall(DMLabelSetValue(label, join[0], edgeSets[e]));
3159     PetscCall(DMPlexRestoreJoin(dm, 2, verts, &njoin, &join));
3160   }
3161   PetscCall(PetscFree(edges));
3162   PetscCall(PetscFree(edgeSets));
3163   if (tps_distribute) {
3164     DM               pdm = NULL;
3165     PetscPartitioner part;
3166 
3167     PetscCall(DMPlexGetPartitioner(dm, &part));
3168     PetscCall(PetscPartitionerSetFromOptions(part));
3169     PetscCall(DMPlexDistribute(dm, 0, NULL, &pdm));
3170     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
3171     // Do not auto-distribute again
3172     PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
3173   }
3174 
3175   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3176   for (PetscInt refine = 0; refine < refinements; refine++) {
3177     PetscInt     m;
3178     DM           dmf;
3179     Vec          X;
3180     PetscScalar *x;
3181     PetscCall(DMRefine(dm, MPI_COMM_NULL, &dmf));
3182     PetscCall(DMPlexReplace_Internal(dm, &dmf));
3183 
3184     PetscCall(DMGetCoordinatesLocal(dm, &X));
3185     PetscCall(VecGetLocalSize(X, &m));
3186     PetscCall(VecGetArray(X, &x));
3187     for (PetscInt i = 0; i < m; i += 3) PetscCall(TPSNearestPoint(evalFunc, &x[i]));
3188     PetscCall(VecRestoreArray(X, &x));
3189   }
3190 
3191   // Face Sets has already been propagated to new vertices during refinement; this propagates to the initial vertices.
3192   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3193   PetscCall(DMPlexLabelComplete(dm, label));
3194 
3195   if (thickness > 0) {
3196     DM              edm, cdm, ecdm;
3197     DMPlexTransform tr;
3198     const char     *prefix;
3199     PetscOptions    options;
3200     // Code from DMPlexExtrude
3201     PetscCall(DMPlexTransformCreate(PetscObjectComm((PetscObject)dm), &tr));
3202     PetscCall(DMPlexTransformSetDM(tr, dm));
3203     PetscCall(DMPlexTransformSetType(tr, DMPLEXEXTRUDE));
3204     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
3205     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)tr, prefix));
3206     PetscCall(PetscObjectGetOptions((PetscObject)dm, &options));
3207     PetscCall(PetscObjectSetOptions((PetscObject)tr, options));
3208     PetscCall(DMPlexTransformExtrudeSetLayers(tr, layers));
3209     PetscCall(DMPlexTransformExtrudeSetThickness(tr, thickness));
3210     PetscCall(DMPlexTransformExtrudeSetTensor(tr, PETSC_FALSE));
3211     PetscCall(DMPlexTransformExtrudeSetSymmetric(tr, PETSC_TRUE));
3212     PetscCall(DMPlexTransformExtrudeSetNormalFunction(tr, normalFunc));
3213     PetscCall(DMPlexTransformSetFromOptions(tr));
3214     PetscCall(PetscObjectSetOptions((PetscObject)tr, NULL));
3215     PetscCall(DMPlexTransformSetUp(tr));
3216     PetscCall(PetscObjectViewFromOptions((PetscObject)tr, NULL, "-dm_plex_tps_transform_view"));
3217     PetscCall(DMPlexTransformApply(tr, dm, &edm));
3218     PetscCall(DMCopyDisc(dm, edm));
3219     PetscCall(DMGetCoordinateDM(dm, &cdm));
3220     PetscCall(DMGetCoordinateDM(edm, &ecdm));
3221     PetscCall(DMCopyDisc(cdm, ecdm));
3222     PetscCall(DMPlexTransformCreateDiscLabels(tr, edm));
3223     PetscCall(DMPlexTransformDestroy(&tr));
3224     if (edm) {
3225       ((DM_Plex *)edm->data)->printFEM    = ((DM_Plex *)dm->data)->printFEM;
3226       ((DM_Plex *)edm->data)->printL2     = ((DM_Plex *)dm->data)->printL2;
3227       ((DM_Plex *)edm->data)->printLocate = ((DM_Plex *)dm->data)->printLocate;
3228     }
3229     PetscCall(DMPlexReplace_Internal(dm, &edm));
3230   }
3231   PetscFunctionReturn(0);
3232 }
3233 
3234 /*@
3235   DMPlexCreateTPSMesh - Create a distributed, interpolated mesh of a triply-periodic surface
3236 
3237   Collective
3238 
3239   Input Parameters:
3240 + comm   - The communicator for the `DM` object
3241 . tpstype - Type of triply-periodic surface
3242 . extent - Array of length 3 containing number of periods in each direction
3243 . periodic - array of length 3 with periodicity, or NULL for non-periodic
3244 . tps_distribute - Distribute 2D manifold mesh prior to refinement and extrusion (more scalable)
3245 . refinements - Number of factor-of-2 refinements of 2D manifold mesh
3246 . layers - Number of cell layers extruded in normal direction
3247 - thickness - Thickness in normal direction
3248 
3249   Output Parameter:
3250 . dm  - The `DM` object
3251 
3252   Level: beginner
3253 
3254   Notes:
3255   This meshes the surface of the Schwarz P or Gyroid surfaces.  Schwarz P is is the simplest member of the triply-periodic minimal surfaces.
3256   https://en.wikipedia.org/wiki/Schwarz_minimal_surface#Schwarz_P_(%22Primitive%22) and can be cut with "clean" boundaries.
3257   The Gyroid (https://en.wikipedia.org/wiki/Gyroid) is another triply-periodic minimal surface with applications in additive manufacturing; it is much more difficult to "cut" since there are no planes of symmetry.
3258   Our implementation creates a very coarse mesh of the surface and refines (by 4-way splitting) as many times as requested.
3259   On each refinement, all vertices are projected to their nearest point on the surface.
3260   This projection could readily be extended to related surfaces.
3261 
3262   The face (edge) sets for the Schwarz P surface are numbered 1(-x), 2(+x), 3(-y), 4(+y), 5(-z), 6(+z).
3263   When the mesh is refined, "Face Sets" contain the new vertices (created during refinement).  Use DMPlexLabelComplete() to propagate to coarse-level vertices.
3264 
3265   Developer Note:
3266   The Gyroid mesh does not currently mark boundary sets.
3267 
3268   References:
3269 . * - Maskery et al, Insights into the mechanical properties of several triply periodic minimal surface lattice structures made by polymer additive manufacturing, 2017.
3270     https://doi.org/10.1016/j.polymer.2017.11.049
3271 
3272 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMSetType()`, `DMCreate()`
3273 @*/
3274 PetscErrorCode DMPlexCreateTPSMesh(MPI_Comm comm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness, DM *dm)
3275 {
3276   PetscFunctionBegin;
3277   PetscCall(DMCreate(comm, dm));
3278   PetscCall(DMSetType(*dm, DMPLEX));
3279   PetscCall(DMPlexCreateTPSMesh_Internal(*dm, tpstype, extent, periodic, tps_distribute, refinements, layers, thickness));
3280   PetscFunctionReturn(0);
3281 }
3282 
3283 /*@
3284   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
3285 
3286   Collective
3287 
3288   Input Parameters:
3289 + comm    - The communicator for the `DM` object
3290 . dim     - The dimension
3291 . simplex - Use simplices, or tensor product cells
3292 - R       - The radius
3293 
3294   Output Parameter:
3295 . dm  - The `DM` object
3296 
3297   Level: beginner
3298 
3299 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBallMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3300 @*/
3301 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscReal R, DM *dm)
3302 {
3303   PetscFunctionBegin;
3304   PetscValidPointer(dm, 5);
3305   PetscCall(DMCreate(comm, dm));
3306   PetscCall(DMSetType(*dm, DMPLEX));
3307   PetscCall(DMPlexCreateSphereMesh_Internal(*dm, dim, simplex, R));
3308   PetscFunctionReturn(0);
3309 }
3310 
3311 static PetscErrorCode DMPlexCreateBallMesh_Internal(DM dm, PetscInt dim, PetscReal R)
3312 {
3313   DM      sdm, vol;
3314   DMLabel bdlabel;
3315 
3316   PetscFunctionBegin;
3317   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
3318   PetscCall(DMSetType(sdm, DMPLEX));
3319   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sdm, "bd_"));
3320   PetscCall(DMPlexCreateSphereMesh_Internal(sdm, dim - 1, PETSC_TRUE, R));
3321   PetscCall(DMSetFromOptions(sdm));
3322   PetscCall(DMViewFromOptions(sdm, NULL, "-dm_view"));
3323   PetscCall(DMPlexGenerate(sdm, NULL, PETSC_TRUE, &vol));
3324   PetscCall(DMDestroy(&sdm));
3325   PetscCall(DMPlexReplace_Internal(dm, &vol));
3326   PetscCall(DMCreateLabel(dm, "marker"));
3327   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
3328   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
3329   PetscCall(DMPlexLabelComplete(dm, bdlabel));
3330   PetscFunctionReturn(0);
3331 }
3332 
3333 /*@
3334   DMPlexCreateBallMesh - Creates a simplex mesh on the d-dimensional ball, B^d.
3335 
3336   Collective
3337 
3338   Input Parameters:
3339 + comm  - The communicator for the `DM` object
3340 . dim   - The dimension
3341 - R     - The radius
3342 
3343   Output Parameter:
3344 . dm  - The `DM` object
3345 
3346   Options Database Key:
3347 - bd_dm_refine - This will refine the surface mesh preserving the sphere geometry
3348 
3349   Level: beginner
3350 
3351 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3352 @*/
3353 PetscErrorCode DMPlexCreateBallMesh(MPI_Comm comm, PetscInt dim, PetscReal R, DM *dm)
3354 {
3355   PetscFunctionBegin;
3356   PetscCall(DMCreate(comm, dm));
3357   PetscCall(DMSetType(*dm, DMPLEX));
3358   PetscCall(DMPlexCreateBallMesh_Internal(*dm, dim, R));
3359   PetscFunctionReturn(0);
3360 }
3361 
3362 static PetscErrorCode DMPlexCreateReferenceCell_Internal(DM rdm, DMPolytopeType ct)
3363 {
3364   PetscFunctionBegin;
3365   switch (ct) {
3366   case DM_POLYTOPE_POINT: {
3367     PetscInt    numPoints[1]        = {1};
3368     PetscInt    coneSize[1]         = {0};
3369     PetscInt    cones[1]            = {0};
3370     PetscInt    coneOrientations[1] = {0};
3371     PetscScalar vertexCoords[1]     = {0.0};
3372 
3373     PetscCall(DMSetDimension(rdm, 0));
3374     PetscCall(DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3375   } break;
3376   case DM_POLYTOPE_SEGMENT: {
3377     PetscInt    numPoints[2]        = {2, 1};
3378     PetscInt    coneSize[3]         = {2, 0, 0};
3379     PetscInt    cones[2]            = {1, 2};
3380     PetscInt    coneOrientations[2] = {0, 0};
3381     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3382 
3383     PetscCall(DMSetDimension(rdm, 1));
3384     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3385   } break;
3386   case DM_POLYTOPE_POINT_PRISM_TENSOR: {
3387     PetscInt    numPoints[2]        = {2, 1};
3388     PetscInt    coneSize[3]         = {2, 0, 0};
3389     PetscInt    cones[2]            = {1, 2};
3390     PetscInt    coneOrientations[2] = {0, 0};
3391     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3392 
3393     PetscCall(DMSetDimension(rdm, 1));
3394     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3395   } break;
3396   case DM_POLYTOPE_TRIANGLE: {
3397     PetscInt    numPoints[2]        = {3, 1};
3398     PetscInt    coneSize[4]         = {3, 0, 0, 0};
3399     PetscInt    cones[3]            = {1, 2, 3};
3400     PetscInt    coneOrientations[3] = {0, 0, 0};
3401     PetscScalar vertexCoords[6]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0};
3402 
3403     PetscCall(DMSetDimension(rdm, 2));
3404     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3405   } break;
3406   case DM_POLYTOPE_QUADRILATERAL: {
3407     PetscInt    numPoints[2]        = {4, 1};
3408     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3409     PetscInt    cones[4]            = {1, 2, 3, 4};
3410     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3411     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0};
3412 
3413     PetscCall(DMSetDimension(rdm, 2));
3414     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3415   } break;
3416   case DM_POLYTOPE_SEG_PRISM_TENSOR: {
3417     PetscInt    numPoints[2]        = {4, 1};
3418     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3419     PetscInt    cones[4]            = {1, 2, 3, 4};
3420     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3421     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0};
3422 
3423     PetscCall(DMSetDimension(rdm, 2));
3424     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3425   } break;
3426   case DM_POLYTOPE_TETRAHEDRON: {
3427     PetscInt    numPoints[2]        = {4, 1};
3428     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3429     PetscInt    cones[4]            = {1, 2, 3, 4};
3430     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3431     PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0};
3432 
3433     PetscCall(DMSetDimension(rdm, 3));
3434     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3435   } break;
3436   case DM_POLYTOPE_HEXAHEDRON: {
3437     PetscInt    numPoints[2]        = {8, 1};
3438     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3439     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3440     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3441     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3442 
3443     PetscCall(DMSetDimension(rdm, 3));
3444     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3445   } break;
3446   case DM_POLYTOPE_TRI_PRISM: {
3447     PetscInt    numPoints[2]        = {6, 1};
3448     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3449     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3450     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3451     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3452 
3453     PetscCall(DMSetDimension(rdm, 3));
3454     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3455   } break;
3456   case DM_POLYTOPE_TRI_PRISM_TENSOR: {
3457     PetscInt    numPoints[2]        = {6, 1};
3458     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3459     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3460     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3461     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3462 
3463     PetscCall(DMSetDimension(rdm, 3));
3464     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3465   } break;
3466   case DM_POLYTOPE_QUAD_PRISM_TENSOR: {
3467     PetscInt    numPoints[2]        = {8, 1};
3468     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3469     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3470     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3471     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3472 
3473     PetscCall(DMSetDimension(rdm, 3));
3474     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3475   } break;
3476   case DM_POLYTOPE_PYRAMID: {
3477     PetscInt    numPoints[2]        = {5, 1};
3478     PetscInt    coneSize[6]         = {5, 0, 0, 0, 0, 0};
3479     PetscInt    cones[5]            = {1, 2, 3, 4, 5};
3480     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3481     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, 0.0, 0.0, 1.0};
3482 
3483     PetscCall(DMSetDimension(rdm, 3));
3484     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3485   } break;
3486   default:
3487     SETERRQ(PetscObjectComm((PetscObject)rdm), PETSC_ERR_ARG_WRONG, "Cannot create reference cell for cell type %s", DMPolytopeTypes[ct]);
3488   }
3489   {
3490     PetscInt Nv, v;
3491 
3492     /* Must create the celltype label here so that we do not automatically try to compute the types */
3493     PetscCall(DMCreateLabel(rdm, "celltype"));
3494     PetscCall(DMPlexSetCellType(rdm, 0, ct));
3495     PetscCall(DMPlexGetChart(rdm, NULL, &Nv));
3496     for (v = 1; v < Nv; ++v) PetscCall(DMPlexSetCellType(rdm, v, DM_POLYTOPE_POINT));
3497   }
3498   PetscCall(DMPlexInterpolateInPlace_Internal(rdm));
3499   PetscCall(PetscObjectSetName((PetscObject)rdm, DMPolytopeTypes[ct]));
3500   PetscFunctionReturn(0);
3501 }
3502 
3503 /*@
3504   DMPlexCreateReferenceCell - Create a `DMPLEX` with the appropriate FEM reference cell
3505 
3506   Collective
3507 
3508   Input Parameters:
3509 + comm - The communicator
3510 - ct   - The cell type of the reference cell
3511 
3512   Output Parameter:
3513 . refdm - The reference cell
3514 
3515   Level: intermediate
3516 
3517 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateReferenceCell()`, `DMPlexCreateBoxMesh()`
3518 @*/
3519 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, DMPolytopeType ct, DM *refdm)
3520 {
3521   PetscFunctionBegin;
3522   PetscCall(DMCreate(comm, refdm));
3523   PetscCall(DMSetType(*refdm, DMPLEX));
3524   PetscCall(DMPlexCreateReferenceCell_Internal(*refdm, ct));
3525   PetscFunctionReturn(0);
3526 }
3527 
3528 static PetscErrorCode DMPlexCreateBoundaryLabel_Private(DM dm, const char name[])
3529 {
3530   DM        plex;
3531   DMLabel   label;
3532   PetscBool hasLabel;
3533 
3534   PetscFunctionBegin;
3535   PetscCall(DMHasLabel(dm, name, &hasLabel));
3536   if (hasLabel) PetscFunctionReturn(0);
3537   PetscCall(DMCreateLabel(dm, name));
3538   PetscCall(DMGetLabel(dm, name, &label));
3539   PetscCall(DMConvert(dm, DMPLEX, &plex));
3540   PetscCall(DMPlexMarkBoundaryFaces(plex, 1, label));
3541   PetscCall(DMPlexLabelComplete(plex, label));
3542   PetscCall(DMDestroy(&plex));
3543   PetscFunctionReturn(0);
3544 }
3545 
3546 /*
3547   We use the last coordinate as the radius, the inner radius is lower[dim-1] and the outer radius is upper[dim-1]. Then we map the first coordinate around the circle.
3548 
3549     (x, y) -> (r, theta) = (x[1], (x[0] - lower[0]) * 2\pi/(upper[0] - lower[0]))
3550 */
3551 static void boxToAnnulus(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
3552 {
3553   const PetscReal low = PetscRealPart(constants[0]);
3554   const PetscReal upp = PetscRealPart(constants[1]);
3555   const PetscReal r   = PetscRealPart(u[1]);
3556   const PetscReal th  = 2. * PETSC_PI * (PetscRealPart(u[0]) - low) / (upp - low);
3557 
3558   f0[0] = r * PetscCosReal(th);
3559   f0[1] = r * PetscSinReal(th);
3560 }
3561 
3562 const char *const DMPlexShapes[] = {"box", "box_surface", "ball", "sphere", "cylinder", "schwarz_p", "gyroid", "doublet", "annulus", "unknown", "DMPlexShape", "DM_SHAPE_", NULL};
3563 
3564 static PetscErrorCode DMPlexCreateFromOptions_Internal(PetscOptionItems *PetscOptionsObject, PetscBool *useCoordSpace, DM dm)
3565 {
3566   DMPlexShape    shape   = DM_SHAPE_BOX;
3567   DMPolytopeType cell    = DM_POLYTOPE_TRIANGLE;
3568   PetscInt       dim     = 2;
3569   PetscBool      simplex = PETSC_TRUE, interpolate = PETSC_TRUE, adjCone = PETSC_FALSE, adjClosure = PETSC_TRUE, refDomain = PETSC_FALSE;
3570   PetscBool      flg, flg2, fflg, bdfflg, nameflg;
3571   MPI_Comm       comm;
3572   char           filename[PETSC_MAX_PATH_LEN]   = "<unspecified>";
3573   char           bdFilename[PETSC_MAX_PATH_LEN] = "<unspecified>";
3574   char           plexname[PETSC_MAX_PATH_LEN]   = "";
3575 
3576   PetscFunctionBegin;
3577   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3578   /* TODO Turn this into a registration interface */
3579   PetscCall(PetscOptionsString("-dm_plex_filename", "File containing a mesh", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &fflg));
3580   PetscCall(PetscOptionsString("-dm_plex_boundary_filename", "File containing a mesh boundary", "DMPlexCreateFromFile", bdFilename, bdFilename, sizeof(bdFilename), &bdfflg));
3581   PetscCall(PetscOptionsString("-dm_plex_name", "Name of the mesh in the file", "DMPlexCreateFromFile", plexname, plexname, sizeof(plexname), &nameflg));
3582   PetscCall(PetscOptionsEnum("-dm_plex_cell", "Cell shape", "", DMPolytopeTypes, (PetscEnum)cell, (PetscEnum *)&cell, NULL));
3583   PetscCall(PetscOptionsBool("-dm_plex_reference_cell_domain", "Use a reference cell domain", "", refDomain, &refDomain, NULL));
3584   PetscCall(PetscOptionsEnum("-dm_plex_shape", "Shape for built-in mesh", "", DMPlexShapes, (PetscEnum)shape, (PetscEnum *)&shape, &flg));
3585   PetscCall(PetscOptionsBoundedInt("-dm_plex_dim", "Topological dimension of the mesh", "DMGetDimension", dim, &dim, &flg, 0));
3586   PetscCheck(!(dim < 0) && !(dim > 3), comm, PETSC_ERR_ARG_OUTOFRANGE, "Dimension %" PetscInt_FMT " should be in [1, 3]", dim);
3587   PetscCall(PetscOptionsBool("-dm_plex_simplex", "Mesh cell shape", "", simplex, &simplex, &flg));
3588   PetscCall(PetscOptionsBool("-dm_plex_interpolate", "Flag to create edges and faces automatically", "", interpolate, &interpolate, &flg));
3589   PetscCall(PetscOptionsBool("-dm_plex_adj_cone", "Set adjacency direction", "DMSetBasicAdjacency", adjCone, &adjCone, &flg));
3590   PetscCall(PetscOptionsBool("-dm_plex_adj_closure", "Set adjacency size", "DMSetBasicAdjacency", adjClosure, &adjClosure, &flg2));
3591   if (flg || flg2) PetscCall(DMSetBasicAdjacency(dm, adjCone, adjClosure));
3592 
3593   switch (cell) {
3594   case DM_POLYTOPE_POINT:
3595   case DM_POLYTOPE_SEGMENT:
3596   case DM_POLYTOPE_POINT_PRISM_TENSOR:
3597   case DM_POLYTOPE_TRIANGLE:
3598   case DM_POLYTOPE_QUADRILATERAL:
3599   case DM_POLYTOPE_TETRAHEDRON:
3600   case DM_POLYTOPE_HEXAHEDRON:
3601     *useCoordSpace = PETSC_TRUE;
3602     break;
3603   default:
3604     *useCoordSpace = PETSC_FALSE;
3605     break;
3606   }
3607 
3608   if (fflg) {
3609     DM dmnew;
3610 
3611     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), filename, plexname, interpolate, &dmnew));
3612     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3613     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3614   } else if (refDomain) {
3615     PetscCall(DMPlexCreateReferenceCell_Internal(dm, cell));
3616   } else if (bdfflg) {
3617     DM bdm, dmnew;
3618 
3619     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), bdFilename, plexname, interpolate, &bdm));
3620     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)bdm, "bd_"));
3621     PetscCall(DMSetFromOptions(bdm));
3622     PetscCall(DMPlexGenerate(bdm, NULL, interpolate, &dmnew));
3623     PetscCall(DMDestroy(&bdm));
3624     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3625     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3626   } else {
3627     PetscCall(PetscObjectSetName((PetscObject)dm, DMPlexShapes[shape]));
3628     switch (shape) {
3629     case DM_SHAPE_BOX:
3630     case DM_SHAPE_ANNULUS: {
3631       PetscInt       faces[3]  = {0, 0, 0};
3632       PetscReal      lower[3]  = {0, 0, 0};
3633       PetscReal      upper[3]  = {1, 1, 1};
3634       DMBoundaryType bdt[3]    = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3635       PetscBool      isAnnular = shape == DM_SHAPE_ANNULUS ? PETSC_TRUE : PETSC_FALSE;
3636       PetscInt       i, n;
3637 
3638       n = dim;
3639       for (i = 0; i < dim; ++i) faces[i] = (dim == 1 ? 1 : 4 - dim);
3640       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3641       n = 3;
3642       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3643       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3644       n = 3;
3645       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3646       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3647       n = 3;
3648       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
3649       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3650 
3651       PetscCheck(!isAnnular || dim == 2, comm, PETSC_ERR_ARG_OUTOFRANGE, "Only two dimensional annuli have been implemented");
3652       if (isAnnular)
3653         for (i = 0; i < dim - 1; ++i) bdt[i] = DM_BOUNDARY_PERIODIC;
3654 
3655       switch (cell) {
3656       case DM_POLYTOPE_TRI_PRISM_TENSOR:
3657         PetscCall(DMPlexCreateWedgeBoxMesh_Internal(dm, faces, lower, upper, bdt));
3658         if (!interpolate) {
3659           DM udm;
3660 
3661           PetscCall(DMPlexUninterpolate(dm, &udm));
3662           PetscCall(DMPlexReplace_Internal(dm, &udm));
3663         }
3664         break;
3665       default:
3666         PetscCall(DMPlexCreateBoxMesh_Internal(dm, dim, simplex, faces, lower, upper, bdt, interpolate));
3667         break;
3668       }
3669       if (isAnnular) {
3670         DM          cdm;
3671         PetscDS     cds;
3672         PetscScalar bounds[2] = {lower[0], upper[0]};
3673 
3674         // Fix coordinates for annular region
3675         PetscCall(DMSetPeriodicity(dm, NULL, NULL, NULL));
3676         PetscCall(DMSetCellCoordinatesLocal(dm, NULL));
3677         PetscCall(DMSetCellCoordinates(dm, NULL));
3678         PetscCall(DMPlexCreateCoordinateSpace(dm, 1, NULL));
3679         PetscCall(DMGetCoordinateDM(dm, &cdm));
3680         PetscCall(DMGetDS(cdm, &cds));
3681         PetscCall(PetscDSSetConstants(cds, 2, bounds));
3682         PetscCall(DMPlexRemapGeometry(dm, 0.0, boxToAnnulus));
3683       }
3684     } break;
3685     case DM_SHAPE_BOX_SURFACE: {
3686       PetscInt  faces[3] = {0, 0, 0};
3687       PetscReal lower[3] = {0, 0, 0};
3688       PetscReal upper[3] = {1, 1, 1};
3689       PetscInt  i, n;
3690 
3691       n = dim + 1;
3692       for (i = 0; i < dim + 1; ++i) faces[i] = (dim + 1 == 1 ? 1 : 4 - (dim + 1));
3693       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3694       n = 3;
3695       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3696       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3697       n = 3;
3698       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3699       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3700       PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(dm, dim + 1, faces, lower, upper, interpolate));
3701     } break;
3702     case DM_SHAPE_SPHERE: {
3703       PetscReal R = 1.0;
3704 
3705       PetscCall(PetscOptionsReal("-dm_plex_sphere_radius", "Radius of the sphere", "", R, &R, &flg));
3706       PetscCall(DMPlexCreateSphereMesh_Internal(dm, dim, simplex, R));
3707     } break;
3708     case DM_SHAPE_BALL: {
3709       PetscReal R = 1.0;
3710 
3711       PetscCall(PetscOptionsReal("-dm_plex_ball_radius", "Radius of the ball", "", R, &R, &flg));
3712       PetscCall(DMPlexCreateBallMesh_Internal(dm, dim, R));
3713     } break;
3714     case DM_SHAPE_CYLINDER: {
3715       DMBoundaryType bdt = DM_BOUNDARY_NONE;
3716       PetscInt       Nw  = 6;
3717 
3718       PetscCall(PetscOptionsEnum("-dm_plex_cylinder_bd", "Boundary type in the z direction", "", DMBoundaryTypes, (PetscEnum)bdt, (PetscEnum *)&bdt, NULL));
3719       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_wedges", "Number of wedges around the cylinder", "", Nw, &Nw, NULL));
3720       switch (cell) {
3721       case DM_POLYTOPE_TRI_PRISM_TENSOR:
3722         PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(dm, Nw, interpolate));
3723         break;
3724       default:
3725         PetscCall(DMPlexCreateHexCylinderMesh_Internal(dm, bdt));
3726         break;
3727       }
3728     } break;
3729     case DM_SHAPE_SCHWARZ_P: // fallthrough
3730     case DM_SHAPE_GYROID: {
3731       PetscInt       extent[3] = {1, 1, 1}, refine = 0, layers = 0, three;
3732       PetscReal      thickness   = 0.;
3733       DMBoundaryType periodic[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3734       DMPlexTPSType  tps_type    = shape == DM_SHAPE_SCHWARZ_P ? DMPLEX_TPS_SCHWARZ_P : DMPLEX_TPS_GYROID;
3735       PetscBool      tps_distribute;
3736       PetscCall(PetscOptionsIntArray("-dm_plex_tps_extent", "Number of replicas for each of three dimensions", NULL, extent, (three = 3, &three), NULL));
3737       PetscCall(PetscOptionsInt("-dm_plex_tps_refine", "Number of refinements", NULL, refine, &refine, NULL));
3738       PetscCall(PetscOptionsEnumArray("-dm_plex_tps_periodic", "Periodicity in each of three dimensions", NULL, DMBoundaryTypes, (PetscEnum *)periodic, (three = 3, &three), NULL));
3739       PetscCall(PetscOptionsInt("-dm_plex_tps_layers", "Number of layers in volumetric extrusion (or zero to not extrude)", NULL, layers, &layers, NULL));
3740       PetscCall(PetscOptionsReal("-dm_plex_tps_thickness", "Thickness of volumetric extrusion", NULL, thickness, &thickness, NULL));
3741       PetscCall(DMPlexDistributeGetDefault(dm, &tps_distribute));
3742       PetscCall(PetscOptionsBool("-dm_plex_tps_distribute", "Distribute the 2D mesh prior to refinement and extrusion", NULL, tps_distribute, &tps_distribute, NULL));
3743       PetscCall(DMPlexCreateTPSMesh_Internal(dm, tps_type, extent, periodic, tps_distribute, refine, layers, thickness));
3744     } break;
3745     case DM_SHAPE_DOUBLET: {
3746       DM        dmnew;
3747       PetscReal rl = 0.0;
3748 
3749       PetscCall(PetscOptionsReal("-dm_plex_doublet_refinementlimit", "Refinement limit", NULL, rl, &rl, NULL));
3750       PetscCall(DMPlexCreateDoublet(PetscObjectComm((PetscObject)dm), dim, simplex, interpolate, rl, &dmnew));
3751       PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3752       PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3753     } break;
3754     default:
3755       SETERRQ(comm, PETSC_ERR_SUP, "Domain shape %s is unsupported", DMPlexShapes[shape]);
3756     }
3757   }
3758   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3759   if (!((PetscObject)dm)->name && nameflg) PetscCall(PetscObjectSetName((PetscObject)dm, plexname));
3760   PetscFunctionReturn(0);
3761 }
3762 
3763 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
3764 {
3765   DM_Plex  *mesh = (DM_Plex *)dm->data;
3766   PetscBool flg, flg2;
3767   char      bdLabel[PETSC_MAX_PATH_LEN];
3768 
3769   PetscFunctionBegin;
3770   /* Handle viewing */
3771   PetscCall(PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL));
3772   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fem", "Debug output level all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL, 0));
3773   PetscCall(PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL));
3774   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL, 0));
3775   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_locate", "Debug output level all point location computations", "DMLocatePoints", 0, &mesh->printLocate, NULL, 0));
3776   PetscCall(DMMonitorSetFromOptions(dm, "-dm_plex_monitor_throughput", "Monitor the simulation throughput", "DMPlexMonitorThroughput", DMPlexMonitorThroughput, NULL, &flg));
3777   if (flg) PetscCall(PetscLogDefaultBegin());
3778   /* Labeling */
3779   PetscCall(PetscOptionsString("-dm_plex_boundary_label", "Label to mark the mesh boundary", "", bdLabel, bdLabel, sizeof(bdLabel), &flg));
3780   if (flg) PetscCall(DMPlexCreateBoundaryLabel_Private(dm, bdLabel));
3781   /* Point Location */
3782   PetscCall(PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL));
3783   /* Partitioning and distribution */
3784   PetscCall(PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL));
3785   /* Generation and remeshing */
3786   PetscCall(PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL));
3787   /* Projection behavior */
3788   PetscCall(PetscOptionsBoundedInt("-dm_plex_max_projection_height", "Maximum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL, 0));
3789   PetscCall(PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL));
3790   /* Checking structure */
3791   {
3792     PetscBool all = PETSC_FALSE;
3793 
3794     PetscCall(PetscOptionsBool("-dm_plex_check_all", "Perform all basic checks", "DMPlexCheck", PETSC_FALSE, &all, NULL));
3795     if (all) {
3796       PetscCall(DMPlexCheck(dm));
3797     } else {
3798       PetscCall(PetscOptionsBool("-dm_plex_check_symmetry", "Check that the adjacency information in the mesh is symmetric", "DMPlexCheckSymmetry", PETSC_FALSE, &flg, &flg2));
3799       if (flg && flg2) PetscCall(DMPlexCheckSymmetry(dm));
3800       PetscCall(PetscOptionsBool("-dm_plex_check_skeleton", "Check that each cell has the correct number of vertices (only for homogeneous simplex or tensor meshes)", "DMPlexCheckSkeleton", PETSC_FALSE, &flg, &flg2));
3801       if (flg && flg2) PetscCall(DMPlexCheckSkeleton(dm, 0));
3802       PetscCall(PetscOptionsBool("-dm_plex_check_faces", "Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type", "DMPlexCheckFaces", PETSC_FALSE, &flg, &flg2));
3803       if (flg && flg2) PetscCall(DMPlexCheckFaces(dm, 0));
3804       PetscCall(PetscOptionsBool("-dm_plex_check_geometry", "Check that cells have positive volume", "DMPlexCheckGeometry", PETSC_FALSE, &flg, &flg2));
3805       if (flg && flg2) PetscCall(DMPlexCheckGeometry(dm));
3806       PetscCall(PetscOptionsBool("-dm_plex_check_pointsf", "Check some necessary conditions for PointSF", "DMPlexCheckPointSF", PETSC_FALSE, &flg, &flg2));
3807       if (flg && flg2) PetscCall(DMPlexCheckPointSF(dm, NULL, PETSC_FALSE));
3808       PetscCall(PetscOptionsBool("-dm_plex_check_interface_cones", "Check points on inter-partition interfaces have conforming order of cone points", "DMPlexCheckInterfaceCones", PETSC_FALSE, &flg, &flg2));
3809       if (flg && flg2) PetscCall(DMPlexCheckInterfaceCones(dm));
3810     }
3811     PetscCall(PetscOptionsBool("-dm_plex_check_cell_shape", "Check cell shape", "DMPlexCheckCellShape", PETSC_FALSE, &flg, &flg2));
3812     if (flg && flg2) PetscCall(DMPlexCheckCellShape(dm, PETSC_TRUE, PETSC_DETERMINE));
3813   }
3814   {
3815     PetscReal scale = 1.0;
3816 
3817     PetscCall(PetscOptionsReal("-dm_plex_scale", "Scale factor for mesh coordinates", "DMPlexScale", scale, &scale, &flg));
3818     if (flg) {
3819       Vec coordinates, coordinatesLocal;
3820 
3821       PetscCall(DMGetCoordinates(dm, &coordinates));
3822       PetscCall(DMGetCoordinatesLocal(dm, &coordinatesLocal));
3823       PetscCall(VecScale(coordinates, scale));
3824       PetscCall(VecScale(coordinatesLocal, scale));
3825     }
3826   }
3827   PetscCall(PetscPartitionerSetFromOptions(mesh->partitioner));
3828   PetscFunctionReturn(0);
3829 }
3830 
3831 PetscErrorCode DMSetFromOptions_Overlap_Plex(DM dm, PetscOptionItems *PetscOptionsObject, PetscInt *overlap)
3832 {
3833   PetscInt  numOvLabels = 16, numOvExLabels = 16;
3834   char     *ovLabelNames[16], *ovExLabelNames[16];
3835   PetscInt  numOvValues = 16, numOvExValues = 16, l;
3836   PetscBool flg;
3837 
3838   PetscFunctionBegin;
3839   PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", *overlap, overlap, NULL, 0));
3840   PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_labels", "List of overlap label names", "DMPlexDistribute", ovLabelNames, &numOvLabels, &flg));
3841   if (!flg) numOvLabels = 0;
3842   if (numOvLabels) {
3843     ((DM_Plex *)dm->data)->numOvLabels = numOvLabels;
3844     for (l = 0; l < numOvLabels; ++l) {
3845       PetscCall(DMGetLabel(dm, ovLabelNames[l], &((DM_Plex *)dm->data)->ovLabels[l]));
3846       PetscCheck(((DM_Plex *)dm->data)->ovLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovLabelNames[l]);
3847       PetscCall(PetscFree(ovLabelNames[l]));
3848     }
3849     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_values", "List of overlap label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovValues, &numOvValues, &flg));
3850     if (!flg) numOvValues = 0;
3851     PetscCheck(numOvLabels == numOvValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvLabels, numOvValues);
3852 
3853     PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_exclude_labels", "List of overlap exclude label names", "DMPlexDistribute", ovExLabelNames, &numOvExLabels, &flg));
3854     if (!flg) numOvExLabels = 0;
3855     ((DM_Plex *)dm->data)->numOvExLabels = numOvExLabels;
3856     for (l = 0; l < numOvExLabels; ++l) {
3857       PetscCall(DMGetLabel(dm, ovExLabelNames[l], &((DM_Plex *)dm->data)->ovExLabels[l]));
3858       PetscCheck(((DM_Plex *)dm->data)->ovExLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovExLabelNames[l]);
3859       PetscCall(PetscFree(ovExLabelNames[l]));
3860     }
3861     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_exclude_values", "List of overlap exclude label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovExValues, &numOvExValues, &flg));
3862     if (!flg) numOvExValues = 0;
3863     PetscCheck(numOvExLabels == numOvExValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of exclude labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvExLabels, numOvExValues);
3864   }
3865   PetscFunctionReturn(0);
3866 }
3867 
3868 static PetscErrorCode DMSetFromOptions_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
3869 {
3870   PetscFunctionList        ordlist;
3871   char                     oname[256];
3872   PetscReal                volume    = -1.0;
3873   PetscInt                 prerefine = 0, refine = 0, r, coarsen = 0, overlap = 0, extLayers = 0, dim;
3874   PetscBool                uniformOrig, created = PETSC_FALSE, uniform = PETSC_TRUE, distribute, interpolate = PETSC_TRUE, coordSpace = PETSC_TRUE, remap = PETSC_TRUE, ghostCells = PETSC_FALSE, isHierarchy, ignoreModel = PETSC_FALSE, flg;
3875   DMPlexReorderDefaultFlag reorder;
3876 
3877   PetscFunctionBegin;
3878   PetscOptionsHeadBegin(PetscOptionsObject, "DMPlex Options");
3879   if (dm->cloneOpts) goto non_refine;
3880   /* Handle automatic creation */
3881   PetscCall(DMGetDimension(dm, &dim));
3882   if (dim < 0) {
3883     PetscCall(DMPlexCreateFromOptions_Internal(PetscOptionsObject, &coordSpace, dm));
3884     created = PETSC_TRUE;
3885   }
3886   PetscCall(DMGetDimension(dm, &dim));
3887   /* Handle interpolation before distribution */
3888   PetscCall(PetscOptionsBool("-dm_plex_interpolate_pre", "Flag to interpolate mesh before distribution", "", interpolate, &interpolate, &flg));
3889   if (flg) {
3890     DMPlexInterpolatedFlag interpolated;
3891 
3892     PetscCall(DMPlexIsInterpolated(dm, &interpolated));
3893     if (interpolated == DMPLEX_INTERPOLATED_FULL && !interpolate) {
3894       DM udm;
3895 
3896       PetscCall(DMPlexUninterpolate(dm, &udm));
3897       PetscCall(DMPlexReplace_Internal(dm, &udm));
3898     } else if (interpolated != DMPLEX_INTERPOLATED_FULL && interpolate) {
3899       DM idm;
3900 
3901       PetscCall(DMPlexInterpolate(dm, &idm));
3902       PetscCall(DMPlexReplace_Internal(dm, &idm));
3903     }
3904   }
3905   /* Handle DMPlex refinement before distribution */
3906   PetscCall(PetscOptionsBool("-dm_refine_ignore_model", "Flag to ignore the geometry model when refining", "DMCreate", ignoreModel, &ignoreModel, &flg));
3907   if (flg) ((DM_Plex *)dm->data)->ignoreModel = ignoreModel;
3908   PetscCall(DMPlexGetRefinementUniform(dm, &uniformOrig));
3909   PetscCall(PetscOptionsBoundedInt("-dm_refine_pre", "The number of refinements before distribution", "DMCreate", prerefine, &prerefine, NULL, 0));
3910   PetscCall(PetscOptionsBool("-dm_refine_remap_pre", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
3911   PetscCall(PetscOptionsBool("-dm_refine_uniform_pre", "Flag for uniform refinement before distribution", "DMCreate", uniform, &uniform, &flg));
3912   if (flg) PetscCall(DMPlexSetRefinementUniform(dm, uniform));
3913   PetscCall(PetscOptionsReal("-dm_refine_volume_limit_pre", "The maximum cell volume after refinement before distribution", "DMCreate", volume, &volume, &flg));
3914   if (flg) {
3915     PetscCall(DMPlexSetRefinementUniform(dm, PETSC_FALSE));
3916     PetscCall(DMPlexSetRefinementLimit(dm, volume));
3917     prerefine = PetscMax(prerefine, 1);
3918   }
3919   for (r = 0; r < prerefine; ++r) {
3920     DM             rdm;
3921     PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
3922 
3923     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3924     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
3925     PetscCall(DMPlexReplace_Internal(dm, &rdm));
3926     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3927     if (coordFunc && remap) {
3928       PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
3929       ((DM_Plex *)dm->data)->coordFunc = coordFunc;
3930     }
3931   }
3932   PetscCall(DMPlexSetRefinementUniform(dm, uniformOrig));
3933   /* Handle DMPlex extrusion before distribution */
3934   PetscCall(PetscOptionsBoundedInt("-dm_extrude", "The number of layers to extrude", "", extLayers, &extLayers, NULL, 0));
3935   if (extLayers) {
3936     DM edm;
3937 
3938     PetscCall(DMExtrude(dm, extLayers, &edm));
3939     PetscCall(DMPlexReplace_Internal(dm, &edm));
3940     ((DM_Plex *)dm->data)->coordFunc = NULL;
3941     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3942     extLayers = 0;
3943     PetscCall(DMGetDimension(dm, &dim));
3944   }
3945   /* Handle DMPlex reordering before distribution */
3946   PetscCall(DMPlexReorderGetDefault(dm, &reorder));
3947   PetscCall(MatGetOrderingList(&ordlist));
3948   PetscCall(PetscStrncpy(oname, MATORDERINGNATURAL, sizeof(oname)));
3949   PetscCall(PetscOptionsFList("-dm_plex_reorder", "Set mesh reordering type", "DMPlexGetOrdering", ordlist, MATORDERINGNATURAL, oname, sizeof(oname), &flg));
3950   if (reorder == DMPLEX_REORDER_DEFAULT_TRUE || flg) {
3951     DM pdm;
3952     IS perm;
3953 
3954     PetscCall(DMPlexGetOrdering(dm, oname, NULL, &perm));
3955     PetscCall(DMPlexPermute(dm, perm, &pdm));
3956     PetscCall(ISDestroy(&perm));
3957     PetscCall(DMPlexReplace_Internal(dm, &pdm));
3958     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3959   }
3960   /* Handle DMPlex distribution */
3961   PetscCall(DMPlexDistributeGetDefault(dm, &distribute));
3962   PetscCall(PetscOptionsBool("-dm_distribute", "Flag to redistribute a mesh among processes", "DMPlexDistribute", distribute, &distribute, NULL));
3963   PetscCall(DMSetFromOptions_Overlap_Plex(dm, PetscOptionsObject, &overlap));
3964   if (distribute) {
3965     DM               pdm = NULL;
3966     PetscPartitioner part;
3967 
3968     PetscCall(DMPlexGetPartitioner(dm, &part));
3969     PetscCall(PetscPartitionerSetFromOptions(part));
3970     PetscCall(DMPlexDistribute(dm, overlap, NULL, &pdm));
3971     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
3972   }
3973   /* Create coordinate space */
3974   if (created) {
3975     DM_Plex  *mesh   = (DM_Plex *)dm->data;
3976     PetscInt  degree = 1;
3977     PetscInt  height = 0;
3978     DM        cdm;
3979     PetscBool flg;
3980 
3981     PetscCall(PetscOptionsBool("-dm_coord_space", "Use an FEM space for coordinates", "", coordSpace, &coordSpace, &flg));
3982     PetscCall(PetscOptionsInt("-dm_coord_petscspace_degree", "FEM degree for coordinate space", "", degree, &degree, NULL));
3983     if (coordSpace) PetscCall(DMPlexCreateCoordinateSpace(dm, degree, mesh->coordFunc));
3984     PetscCall(DMGetCoordinateDM(dm, &cdm));
3985     if (flg && !coordSpace) {
3986       PetscDS      cds;
3987       PetscObject  obj;
3988       PetscClassId id;
3989 
3990       PetscCall(DMGetDS(cdm, &cds));
3991       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
3992       PetscCall(PetscObjectGetClassId(obj, &id));
3993       if (id == PETSCFE_CLASSID) {
3994         PetscContainer dummy;
3995 
3996         PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &dummy));
3997         PetscCall(PetscObjectSetName((PetscObject)dummy, "coordinates"));
3998         PetscCall(DMSetField(cdm, 0, NULL, (PetscObject)dummy));
3999         PetscCall(PetscContainerDestroy(&dummy));
4000         PetscCall(DMClearDS(cdm));
4001       }
4002       mesh->coordFunc = NULL;
4003     }
4004     PetscCall(PetscOptionsBool("-dm_sparse_localize", "Localize only necessary cells", "", dm->sparseLocalize, &dm->sparseLocalize, &flg));
4005     PetscCall(PetscOptionsInt("-dm_localize_height", "Localize edges and faces in addition to cells", "", height, &height, &flg));
4006     if (flg) PetscCall(DMPlexSetMaxProjectionHeight(cdm, height));
4007     PetscCall(DMLocalizeCoordinates(dm));
4008   }
4009   /* Handle DMPlex refinement */
4010   remap = PETSC_TRUE;
4011   PetscCall(PetscOptionsBoundedInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL, 0));
4012   PetscCall(PetscOptionsBool("-dm_refine_remap", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
4013   PetscCall(PetscOptionsBoundedInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy, 0));
4014   if (refine) PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
4015   if (refine && isHierarchy) {
4016     DM *dms, coarseDM;
4017 
4018     PetscCall(DMGetCoarseDM(dm, &coarseDM));
4019     PetscCall(PetscObjectReference((PetscObject)coarseDM));
4020     PetscCall(PetscMalloc1(refine, &dms));
4021     PetscCall(DMRefineHierarchy(dm, refine, dms));
4022     /* Total hack since we do not pass in a pointer */
4023     PetscCall(DMPlexSwap_Static(dm, dms[refine - 1]));
4024     if (refine == 1) {
4025       PetscCall(DMSetCoarseDM(dm, dms[0]));
4026       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4027     } else {
4028       PetscCall(DMSetCoarseDM(dm, dms[refine - 2]));
4029       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4030       PetscCall(DMSetCoarseDM(dms[0], dms[refine - 1]));
4031       PetscCall(DMPlexSetRegularRefinement(dms[0], PETSC_TRUE));
4032     }
4033     PetscCall(DMSetCoarseDM(dms[refine - 1], coarseDM));
4034     PetscCall(PetscObjectDereference((PetscObject)coarseDM));
4035     /* Free DMs */
4036     for (r = 0; r < refine; ++r) {
4037       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4038       PetscCall(DMDestroy(&dms[r]));
4039     }
4040     PetscCall(PetscFree(dms));
4041   } else {
4042     for (r = 0; r < refine; ++r) {
4043       DM             rdm;
4044       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4045 
4046       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4047       PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
4048       /* Total hack since we do not pass in a pointer */
4049       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4050       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4051       if (coordFunc && remap) {
4052         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4053         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4054       }
4055     }
4056   }
4057   /* Handle DMPlex coarsening */
4058   PetscCall(PetscOptionsBoundedInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL, 0));
4059   PetscCall(PetscOptionsBoundedInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy, 0));
4060   if (coarsen && isHierarchy) {
4061     DM *dms;
4062 
4063     PetscCall(PetscMalloc1(coarsen, &dms));
4064     PetscCall(DMCoarsenHierarchy(dm, coarsen, dms));
4065     /* Free DMs */
4066     for (r = 0; r < coarsen; ++r) {
4067       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4068       PetscCall(DMDestroy(&dms[r]));
4069     }
4070     PetscCall(PetscFree(dms));
4071   } else {
4072     for (r = 0; r < coarsen; ++r) {
4073       DM             cdm;
4074       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4075 
4076       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4077       PetscCall(DMCoarsen(dm, PetscObjectComm((PetscObject)dm), &cdm));
4078       /* Total hack since we do not pass in a pointer */
4079       PetscCall(DMPlexReplace_Internal(dm, &cdm));
4080       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4081       if (coordFunc) {
4082         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4083         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4084       }
4085     }
4086   }
4087   /* Handle ghost cells */
4088   PetscCall(PetscOptionsBool("-dm_plex_create_fv_ghost_cells", "Flag to create finite volume ghost cells on the boundary", "DMCreate", ghostCells, &ghostCells, NULL));
4089   if (ghostCells) {
4090     DM   gdm;
4091     char lname[PETSC_MAX_PATH_LEN];
4092 
4093     lname[0] = '\0';
4094     PetscCall(PetscOptionsString("-dm_plex_fv_ghost_cells_label", "Label name for ghost cells boundary", "DMCreate", lname, lname, sizeof(lname), &flg));
4095     PetscCall(DMPlexConstructGhostCells(dm, flg ? lname : NULL, NULL, &gdm));
4096     PetscCall(DMPlexReplace_Internal(dm, &gdm));
4097   }
4098   /* Handle 1D order */
4099   if (reorder != DMPLEX_REORDER_DEFAULT_FALSE && dim == 1) {
4100     DM           cdm, rdm;
4101     PetscDS      cds;
4102     PetscObject  obj;
4103     PetscClassId id = PETSC_OBJECT_CLASSID;
4104     IS           perm;
4105     PetscInt     Nf;
4106     PetscBool    distributed;
4107 
4108     PetscCall(DMPlexIsDistributed(dm, &distributed));
4109     PetscCall(DMGetCoordinateDM(dm, &cdm));
4110     PetscCall(DMGetDS(cdm, &cds));
4111     PetscCall(PetscDSGetNumFields(cds, &Nf));
4112     if (Nf) {
4113       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
4114       PetscCall(PetscObjectGetClassId(obj, &id));
4115     }
4116     if (!distributed && id != PETSCFE_CLASSID) {
4117       PetscCall(DMPlexGetOrdering1D(dm, &perm));
4118       PetscCall(DMPlexPermute(dm, perm, &rdm));
4119       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4120       PetscCall(ISDestroy(&perm));
4121     }
4122   }
4123 /* Handle */
4124 non_refine:
4125   PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4126   PetscOptionsHeadEnd();
4127   PetscFunctionReturn(0);
4128 }
4129 
4130 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm, Vec *vec)
4131 {
4132   PetscFunctionBegin;
4133   PetscCall(DMCreateGlobalVector_Section_Private(dm, vec));
4134   /* PetscCall(VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM)); */
4135   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex));
4136   PetscCall(VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void))VecView_Plex_Native));
4137   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex));
4138   PetscCall(VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void))VecLoad_Plex_Native));
4139   PetscFunctionReturn(0);
4140 }
4141 
4142 static PetscErrorCode DMCreateLocalVector_Plex(DM dm, Vec *vec)
4143 {
4144   PetscFunctionBegin;
4145   PetscCall(DMCreateLocalVector_Section_Private(dm, vec));
4146   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex_Local));
4147   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex_Local));
4148   PetscFunctionReturn(0);
4149 }
4150 
4151 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
4152 {
4153   PetscInt depth, d;
4154 
4155   PetscFunctionBegin;
4156   PetscCall(DMPlexGetDepth(dm, &depth));
4157   if (depth == 1) {
4158     PetscCall(DMGetDimension(dm, &d));
4159     if (dim == 0) PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4160     else if (dim == d) PetscCall(DMPlexGetDepthStratum(dm, 1, pStart, pEnd));
4161     else {
4162       *pStart = 0;
4163       *pEnd   = 0;
4164     }
4165   } else {
4166     PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4167   }
4168   PetscFunctionReturn(0);
4169 }
4170 
4171 static PetscErrorCode DMGetNeighbors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
4172 {
4173   PetscSF            sf;
4174   PetscInt           niranks, njranks, n;
4175   const PetscMPIInt *iranks, *jranks;
4176   DM_Plex           *data = (DM_Plex *)dm->data;
4177 
4178   PetscFunctionBegin;
4179   PetscCall(DMGetPointSF(dm, &sf));
4180   if (!data->neighbors) {
4181     PetscCall(PetscSFSetUp(sf));
4182     PetscCall(PetscSFGetRootRanks(sf, &njranks, &jranks, NULL, NULL, NULL));
4183     PetscCall(PetscSFGetLeafRanks(sf, &niranks, &iranks, NULL, NULL));
4184     PetscCall(PetscMalloc1(njranks + niranks + 1, &data->neighbors));
4185     PetscCall(PetscArraycpy(data->neighbors + 1, jranks, njranks));
4186     PetscCall(PetscArraycpy(data->neighbors + njranks + 1, iranks, niranks));
4187     n = njranks + niranks;
4188     PetscCall(PetscSortRemoveDupsMPIInt(&n, data->neighbors + 1));
4189     /* The following cast should never fail: can't have more neighbors than PETSC_MPI_INT_MAX */
4190     PetscCall(PetscMPIIntCast(n, data->neighbors));
4191   }
4192   if (nranks) *nranks = data->neighbors[0];
4193   if (ranks) {
4194     if (data->neighbors[0]) *ranks = data->neighbors + 1;
4195     else *ranks = NULL;
4196   }
4197   PetscFunctionReturn(0);
4198 }
4199 
4200 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM, DM, Mat, Vec, Vec);
4201 
4202 static PetscErrorCode DMInitialize_Plex(DM dm)
4203 {
4204   PetscFunctionBegin;
4205   dm->ops->view                      = DMView_Plex;
4206   dm->ops->load                      = DMLoad_Plex;
4207   dm->ops->setfromoptions            = DMSetFromOptions_Plex;
4208   dm->ops->clone                     = DMClone_Plex;
4209   dm->ops->setup                     = DMSetUp_Plex;
4210   dm->ops->createlocalsection        = DMCreateLocalSection_Plex;
4211   dm->ops->createdefaultconstraints  = DMCreateDefaultConstraints_Plex;
4212   dm->ops->createglobalvector        = DMCreateGlobalVector_Plex;
4213   dm->ops->createlocalvector         = DMCreateLocalVector_Plex;
4214   dm->ops->getlocaltoglobalmapping   = NULL;
4215   dm->ops->createfieldis             = NULL;
4216   dm->ops->createcoordinatedm        = DMCreateCoordinateDM_Plex;
4217   dm->ops->createcoordinatefield     = DMCreateCoordinateField_Plex;
4218   dm->ops->getcoloring               = NULL;
4219   dm->ops->creatematrix              = DMCreateMatrix_Plex;
4220   dm->ops->createinterpolation       = DMCreateInterpolation_Plex;
4221   dm->ops->createmassmatrix          = DMCreateMassMatrix_Plex;
4222   dm->ops->createmassmatrixlumped    = DMCreateMassMatrixLumped_Plex;
4223   dm->ops->createinjection           = DMCreateInjection_Plex;
4224   dm->ops->refine                    = DMRefine_Plex;
4225   dm->ops->coarsen                   = DMCoarsen_Plex;
4226   dm->ops->refinehierarchy           = DMRefineHierarchy_Plex;
4227   dm->ops->coarsenhierarchy          = DMCoarsenHierarchy_Plex;
4228   dm->ops->extrude                   = DMExtrude_Plex;
4229   dm->ops->globaltolocalbegin        = NULL;
4230   dm->ops->globaltolocalend          = NULL;
4231   dm->ops->localtoglobalbegin        = NULL;
4232   dm->ops->localtoglobalend          = NULL;
4233   dm->ops->destroy                   = DMDestroy_Plex;
4234   dm->ops->createsubdm               = DMCreateSubDM_Plex;
4235   dm->ops->createsuperdm             = DMCreateSuperDM_Plex;
4236   dm->ops->getdimpoints              = DMGetDimPoints_Plex;
4237   dm->ops->locatepoints              = DMLocatePoints_Plex;
4238   dm->ops->projectfunctionlocal      = DMProjectFunctionLocal_Plex;
4239   dm->ops->projectfunctionlabellocal = DMProjectFunctionLabelLocal_Plex;
4240   dm->ops->projectfieldlocal         = DMProjectFieldLocal_Plex;
4241   dm->ops->projectfieldlabellocal    = DMProjectFieldLabelLocal_Plex;
4242   dm->ops->projectbdfieldlabellocal  = DMProjectBdFieldLabelLocal_Plex;
4243   dm->ops->computel2diff             = DMComputeL2Diff_Plex;
4244   dm->ops->computel2gradientdiff     = DMComputeL2GradientDiff_Plex;
4245   dm->ops->computel2fielddiff        = DMComputeL2FieldDiff_Plex;
4246   dm->ops->getneighbors              = DMGetNeighbors_Plex;
4247   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertBoundaryValues_C", DMPlexInsertBoundaryValues_Plex));
4248   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertTimeDerviativeBoundaryValues_C", DMPlexInsertTimeDerivativeBoundaryValues_Plex));
4249   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMSetUpGLVisViewer_C", DMSetUpGLVisViewer_Plex));
4250   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMCreateNeumannOverlap_C", DMCreateNeumannOverlap_Plex));
4251   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4252   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeGetDefault_C", DMPlexDistributeGetDefault_Plex));
4253   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeSetDefault_C", DMPlexDistributeSetDefault_Plex));
4254   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderGetDefault_C", DMPlexReorderGetDefault_Plex));
4255   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderSetDefault_C", DMPlexReorderSetDefault_Plex));
4256   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMInterpolateSolution_C", DMInterpolateSolution_Plex));
4257   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4258   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetOverlap_C", DMPlexSetOverlap_Plex));
4259   PetscFunctionReturn(0);
4260 }
4261 
4262 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
4263 {
4264   DM_Plex *mesh = (DM_Plex *)dm->data;
4265 
4266   PetscFunctionBegin;
4267   mesh->refct++;
4268   (*newdm)->data = mesh;
4269   PetscCall(PetscObjectChangeTypeName((PetscObject)*newdm, DMPLEX));
4270   PetscCall(DMInitialize_Plex(*newdm));
4271   PetscFunctionReturn(0);
4272 }
4273 
4274 /*MC
4275   DMPLEX = "plex" - A `DM` object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
4276                     In the local representation, Vecs contain all unknowns in the interior and shared boundary. This is
4277                     specified by a PetscSection object. Ownership in the global representation is determined by
4278                     ownership of the underlying `DMPLEX` points. This is specified by another `PetscSection` object.
4279 
4280   Options Database Keys:
4281 + -dm_refine_pre                     - Refine mesh before distribution
4282 + -dm_refine_uniform_pre             - Choose uniform or generator-based refinement
4283 + -dm_refine_volume_limit_pre        - Cell volume limit after pre-refinement using generator
4284 . -dm_distribute                     - Distribute mesh across processes
4285 . -dm_distribute_overlap             - Number of cells to overlap for distribution
4286 . -dm_refine                         - Refine mesh after distribution
4287 . -dm_plex_hash_location             - Use grid hashing for point location
4288 . -dm_plex_hash_box_faces <n,m,p>    - The number of divisions in each direction of the grid hash
4289 . -dm_plex_partition_balance         - Attempt to evenly divide points on partition boundary between processes
4290 . -dm_plex_remesh_bd                 - Allow changes to the boundary on remeshing
4291 . -dm_plex_max_projection_height     - Maximum mesh point height used to project locally
4292 . -dm_plex_regular_refinement        - Use special nested projection algorithm for regular refinement
4293 . -dm_plex_check_all                 - Perform all shecks below
4294 . -dm_plex_check_symmetry            - Check that the adjacency information in the mesh is symmetric
4295 . -dm_plex_check_skeleton <celltype> - Check that each cell has the correct number of vertices
4296 . -dm_plex_check_faces <celltype>    - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
4297 . -dm_plex_check_geometry            - Check that cells have positive volume
4298 . -dm_view :mesh.tex:ascii_latex     - View the mesh in LaTeX/TikZ
4299 . -dm_plex_view_scale <num>          - Scale the TikZ
4300 - -dm_plex_print_fem <num>           - View FEM assembly information, such as element vectors and matrices
4301 
4302   Level: intermediate
4303 
4304 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`, `PetscSection`
4305 M*/
4306 
4307 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
4308 {
4309   DM_Plex *mesh;
4310   PetscInt unit;
4311 
4312   PetscFunctionBegin;
4313   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4314   PetscCall(PetscNew(&mesh));
4315   dm->data = mesh;
4316 
4317   mesh->refct = 1;
4318   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection));
4319   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection));
4320   mesh->refinementUniform      = PETSC_TRUE;
4321   mesh->refinementLimit        = -1.0;
4322   mesh->distDefault            = PETSC_TRUE;
4323   mesh->reorderDefault         = DMPLEX_REORDER_DEFAULT_NOTSET;
4324   mesh->distributionName       = NULL;
4325   mesh->interpolated           = DMPLEX_INTERPOLATED_INVALID;
4326   mesh->interpolatedCollective = DMPLEX_INTERPOLATED_INVALID;
4327 
4328   PetscCall(PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner));
4329   mesh->remeshBd = PETSC_FALSE;
4330 
4331   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
4332 
4333   mesh->depthState    = -1;
4334   mesh->celltypeState = -1;
4335   mesh->printTol      = 1.0e-10;
4336 
4337   PetscCall(DMInitialize_Plex(dm));
4338   PetscFunctionReturn(0);
4339 }
4340 
4341 /*@
4342   DMPlexCreate - Creates a `DMPLEX` object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
4343 
4344   Collective
4345 
4346   Input Parameter:
4347 . comm - The communicator for the `DMPLEX` object
4348 
4349   Output Parameter:
4350 . mesh  - The `DMPLEX` object
4351 
4352   Level: beginner
4353 
4354 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`
4355 @*/
4356 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
4357 {
4358   PetscFunctionBegin;
4359   PetscValidPointer(mesh, 2);
4360   PetscCall(DMCreate(comm, mesh));
4361   PetscCall(DMSetType(*mesh, DMPLEX));
4362   PetscFunctionReturn(0);
4363 }
4364 
4365 /*@C
4366   DMPlexBuildFromCellListParallel - Build distributed `DMPLEX` topology from a list of vertices for each cell (common mesh generator output)
4367 
4368   Collective on dm
4369 
4370   Input Parameters:
4371 + dm - The `DM`
4372 . numCells - The number of cells owned by this process
4373 . numVertices - The number of vertices to be owned by this process, or `PETSC_DECIDE`
4374 . NVertices - The global number of vertices, or `PETSC_DETERMINE`
4375 . numCorners - The number of vertices for each cell
4376 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4377 
4378   Output Parameters:
4379 + vertexSF - (Optional) `PetscSF` describing complete vertex ownership
4380 - verticesAdjSaved - (Optional) vertex adjacency array
4381 
4382   Level: advanced
4383 
4384   Notes:
4385   Two triangles sharing a face
4386 .vb
4387 
4388         2
4389       / | \
4390      /  |  \
4391     /   |   \
4392    0  0 | 1  3
4393     \   |   /
4394      \  |  /
4395       \ | /
4396         1
4397 .ve
4398 would have input
4399 .vb
4400   numCells = 2, numVertices = 4
4401   cells = [0 1 2  1 3 2]
4402 .ve
4403 which would result in the `DMPLEX`
4404 .vb
4405 
4406         4
4407       / | \
4408      /  |  \
4409     /   |   \
4410    2  0 | 1  5
4411     \   |   /
4412      \  |  /
4413       \ | /
4414         3
4415 .ve
4416 
4417   Vertices are implicitly numbered consecutively 0,...,NVertices.
4418   Each rank owns a chunk of numVertices consecutive vertices.
4419   If numVertices is `PETSC_DECIDE`, PETSc will distribute them as evenly as possible using PetscLayout.
4420   If NVertices is `PETSC_DETERMINE` and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
4421   If only NVertices is `PETSC_DETERMINE`, it is computed as the sum of numVertices over all ranks.
4422 
4423   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
4424 
4425   Fortran Note:
4426   Not currently supported in Fortran.
4427 
4428 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildCoordinatesFromCellListParallel()`,
4429           `PetscSF`
4430 @*/
4431 PetscErrorCode DMPlexBuildFromCellListParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, const PetscInt cells[], PetscSF *vertexSF, PetscInt **verticesAdjSaved)
4432 {
4433   PetscSF     sfPoint;
4434   PetscLayout layout;
4435   PetscInt    numVerticesAdj, *verticesAdj, *cones, c, p;
4436 
4437   PetscFunctionBegin;
4438   PetscValidLogicalCollectiveInt(dm, NVertices, 4);
4439   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4440   /* Get/check global number of vertices */
4441   {
4442     PetscInt       NVerticesInCells, i;
4443     const PetscInt len = numCells * numCorners;
4444 
4445     /* NVerticesInCells = max(cells) + 1 */
4446     NVerticesInCells = PETSC_MIN_INT;
4447     for (i = 0; i < len; i++)
4448       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4449     ++NVerticesInCells;
4450     PetscCallMPI(MPI_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
4451 
4452     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
4453     else
4454       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
4455   }
4456   /* Count locally unique vertices */
4457   {
4458     PetscHSetI vhash;
4459     PetscInt   off = 0;
4460 
4461     PetscCall(PetscHSetICreate(&vhash));
4462     for (c = 0; c < numCells; ++c) {
4463       for (p = 0; p < numCorners; ++p) PetscCall(PetscHSetIAdd(vhash, cells[c * numCorners + p]));
4464     }
4465     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
4466     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
4467     else verticesAdj = *verticesAdjSaved;
4468     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
4469     PetscCall(PetscHSetIDestroy(&vhash));
4470     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
4471   }
4472   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
4473   /* Create cones */
4474   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
4475   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
4476   PetscCall(DMSetUp(dm));
4477   PetscCall(DMPlexGetCones(dm, &cones));
4478   for (c = 0; c < numCells; ++c) {
4479     for (p = 0; p < numCorners; ++p) {
4480       const PetscInt gv = cells[c * numCorners + p];
4481       PetscInt       lv;
4482 
4483       /* Positions within verticesAdj form 0-based local vertex numbering;
4484          we need to shift it by numCells to get correct DAG points (cells go first) */
4485       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
4486       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
4487       cones[c * numCorners + p] = lv + numCells;
4488     }
4489   }
4490   /* Build point sf */
4491   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
4492   PetscCall(PetscLayoutSetSize(layout, NVertices));
4493   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
4494   PetscCall(PetscLayoutSetBlockSize(layout, 1));
4495   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
4496   PetscCall(PetscLayoutDestroy(&layout));
4497   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
4498   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
4499   if (dm->sf) {
4500     const char *prefix;
4501 
4502     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
4503     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
4504   }
4505   PetscCall(DMSetPointSF(dm, sfPoint));
4506   PetscCall(PetscSFDestroy(&sfPoint));
4507   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)(*vertexSF), "Vertex Ownership SF"));
4508   /* Fill in the rest of the topology structure */
4509   PetscCall(DMPlexSymmetrize(dm));
4510   PetscCall(DMPlexStratify(dm));
4511   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4512   PetscFunctionReturn(0);
4513 }
4514 
4515 /*@C
4516   DMPlexBuildCoordinatesFromCellListParallel - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4517 
4518   Collective on dm
4519 
4520   Input Parameters:
4521 + dm - The `DM`
4522 . spaceDim - The spatial dimension used for coordinates
4523 . sfVert - `PetscSF` describing complete vertex ownership
4524 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4525 
4526   Level: advanced
4527 
4528   Fortran Note:
4529   Not currently supported in Fortran.
4530 
4531 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellListParallel()`
4532 @*/
4533 PetscErrorCode DMPlexBuildCoordinatesFromCellListParallel(DM dm, PetscInt spaceDim, PetscSF sfVert, const PetscReal vertexCoords[])
4534 {
4535   PetscSection coordSection;
4536   Vec          coordinates;
4537   PetscScalar *coords;
4538   PetscInt     numVertices, numVerticesAdj, coordSize, v, vStart, vEnd;
4539 
4540   PetscFunctionBegin;
4541   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4542   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
4543   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4544   PetscCall(DMSetCoordinateDim(dm, spaceDim));
4545   PetscCall(PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL));
4546   PetscCheck(vEnd - vStart == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Supplied sfVert has wrong number of leaves = %" PetscInt_FMT " != %" PetscInt_FMT " = vEnd - vStart", numVerticesAdj, vEnd - vStart);
4547   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4548   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4549   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
4550   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
4551   for (v = vStart; v < vEnd; ++v) {
4552     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
4553     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
4554   }
4555   PetscCall(PetscSectionSetUp(coordSection));
4556   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4557   PetscCall(VecCreate(PetscObjectComm((PetscObject)dm), &coordinates));
4558   PetscCall(VecSetBlockSize(coordinates, spaceDim));
4559   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4560   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4561   PetscCall(VecSetType(coordinates, VECSTANDARD));
4562   PetscCall(VecGetArray(coordinates, &coords));
4563   {
4564     MPI_Datatype coordtype;
4565 
4566     /* Need a temp buffer for coords if we have complex/single */
4567     PetscCallMPI(MPI_Type_contiguous(spaceDim, MPIU_SCALAR, &coordtype));
4568     PetscCallMPI(MPI_Type_commit(&coordtype));
4569 #if defined(PETSC_USE_COMPLEX)
4570     {
4571       PetscScalar *svertexCoords;
4572       PetscInt     i;
4573       PetscCall(PetscMalloc1(numVertices * spaceDim, &svertexCoords));
4574       for (i = 0; i < numVertices * spaceDim; i++) svertexCoords[i] = vertexCoords[i];
4575       PetscCall(PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4576       PetscCall(PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4577       PetscCall(PetscFree(svertexCoords));
4578     }
4579 #else
4580     PetscCall(PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4581     PetscCall(PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4582 #endif
4583     PetscCallMPI(MPI_Type_free(&coordtype));
4584   }
4585   PetscCall(VecRestoreArray(coordinates, &coords));
4586   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4587   PetscCall(VecDestroy(&coordinates));
4588   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4589   PetscFunctionReturn(0);
4590 }
4591 
4592 /*@
4593   DMPlexCreateFromCellListParallelPetsc - Create distributed `DMPLEX` from a list of vertices for each cell (common mesh generator output)
4594 
4595   Collective
4596 
4597   Input Parameters:
4598 + comm - The communicator
4599 . dim - The topological dimension of the mesh
4600 . numCells - The number of cells owned by this process
4601 . numVertices - The number of vertices owned by this process, or `PETSC_DECIDE`
4602 . NVertices - The global number of vertices, or `PETSC_DECIDE`
4603 . numCorners - The number of vertices for each cell
4604 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4605 . cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4606 . spaceDim - The spatial dimension used for coordinates
4607 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4608 
4609   Output Parameters:
4610 + dm - The `DM`
4611 . vertexSF - (Optional) `PetscSF` describing complete vertex ownership
4612 - verticesAdjSaved - (Optional) vertex adjacency array
4613 
4614   Level: intermediate
4615 
4616   Notes:
4617   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`,
4618   `DMPlexBuildFromCellListParallel()`, `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellListParallel()`
4619 
4620   See `DMPlexBuildFromCellListParallel()` for an example and details about the topology-related parameters.
4621 
4622   See `DMPlexBuildCoordinatesFromCellListParallel()` for details about the geometry-related parameters.
4623 
4624 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
4625 @*/
4626 PetscErrorCode DMPlexCreateFromCellListParallelPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, PetscInt **verticesAdj, DM *dm)
4627 {
4628   PetscSF sfVert;
4629 
4630   PetscFunctionBegin;
4631   PetscCall(DMCreate(comm, dm));
4632   PetscCall(DMSetType(*dm, DMPLEX));
4633   PetscValidLogicalCollectiveInt(*dm, dim, 2);
4634   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
4635   PetscCall(DMSetDimension(*dm, dim));
4636   PetscCall(DMPlexBuildFromCellListParallel(*dm, numCells, numVertices, NVertices, numCorners, cells, &sfVert, verticesAdj));
4637   if (interpolate) {
4638     DM idm;
4639 
4640     PetscCall(DMPlexInterpolate(*dm, &idm));
4641     PetscCall(DMDestroy(dm));
4642     *dm = idm;
4643   }
4644   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
4645   if (vertexSF) *vertexSF = sfVert;
4646   else PetscCall(PetscSFDestroy(&sfVert));
4647   PetscFunctionReturn(0);
4648 }
4649 
4650 /*@C
4651   DMPlexBuildFromCellList - Build `DMPLEX` topology from a list of vertices for each cell (common mesh generator output)
4652 
4653   Collective on dm
4654 
4655   Input Parameters:
4656 + dm - The `DM`
4657 . numCells - The number of cells owned by this process
4658 . numVertices - The number of vertices owned by this process, or `PETSC_DETERMINE`
4659 . numCorners - The number of vertices for each cell
4660 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4661 
4662   Level: advanced
4663 
4664   Notes:
4665   Two triangles sharing a face
4666 .vb
4667 
4668         2
4669       / | \
4670      /  |  \
4671     /   |   \
4672    0  0 | 1  3
4673     \   |   /
4674      \  |  /
4675       \ | /
4676         1
4677 .ve
4678 would have input
4679 .vb
4680   numCells = 2, numVertices = 4
4681   cells = [0 1 2  1 3 2]
4682 .ve
4683 which would result in the `DMPLEX`
4684 .vb
4685 
4686         4
4687       / | \
4688      /  |  \
4689     /   |   \
4690    2  0 | 1  5
4691     \   |   /
4692      \  |  /
4693       \ | /
4694         3
4695 .ve
4696 
4697   If numVertices is `PETSC_DETERMINE`, it is computed by PETSc as the maximum vertex index in cells + 1.
4698 
4699   Not currently supported in Fortran.
4700 
4701 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListPetsc()`
4702 @*/
4703 PetscErrorCode DMPlexBuildFromCellList(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const PetscInt cells[])
4704 {
4705   PetscInt *cones, c, p, dim;
4706 
4707   PetscFunctionBegin;
4708   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4709   PetscCall(DMGetDimension(dm, &dim));
4710   /* Get/check global number of vertices */
4711   {
4712     PetscInt       NVerticesInCells, i;
4713     const PetscInt len = numCells * numCorners;
4714 
4715     /* NVerticesInCells = max(cells) + 1 */
4716     NVerticesInCells = PETSC_MIN_INT;
4717     for (i = 0; i < len; i++)
4718       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4719     ++NVerticesInCells;
4720 
4721     if (numVertices == PETSC_DECIDE) numVertices = NVerticesInCells;
4722     else
4723       PetscCheck(numVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, numVertices, NVerticesInCells);
4724   }
4725   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
4726   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
4727   PetscCall(DMSetUp(dm));
4728   PetscCall(DMPlexGetCones(dm, &cones));
4729   for (c = 0; c < numCells; ++c) {
4730     for (p = 0; p < numCorners; ++p) cones[c * numCorners + p] = cells[c * numCorners + p] + numCells;
4731   }
4732   PetscCall(DMPlexSymmetrize(dm));
4733   PetscCall(DMPlexStratify(dm));
4734   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4735   PetscFunctionReturn(0);
4736 }
4737 
4738 /*@C
4739   DMPlexBuildCoordinatesFromCellList - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4740 
4741   Collective on dm
4742 
4743   Input Parameters:
4744 + dm - The `DM`
4745 . spaceDim - The spatial dimension used for coordinates
4746 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4747 
4748   Level: advanced
4749 
4750   Fortran Note:
4751   Not currently supported in Fortran.
4752 
4753 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellList()`
4754 @*/
4755 PetscErrorCode DMPlexBuildCoordinatesFromCellList(DM dm, PetscInt spaceDim, const PetscReal vertexCoords[])
4756 {
4757   PetscSection coordSection;
4758   Vec          coordinates;
4759   DM           cdm;
4760   PetscScalar *coords;
4761   PetscInt     v, vStart, vEnd, d;
4762 
4763   PetscFunctionBegin;
4764   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4765   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
4766   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4767   PetscCall(DMSetCoordinateDim(dm, spaceDim));
4768   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4769   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4770   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
4771   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
4772   for (v = vStart; v < vEnd; ++v) {
4773     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
4774     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
4775   }
4776   PetscCall(PetscSectionSetUp(coordSection));
4777 
4778   PetscCall(DMGetCoordinateDM(dm, &cdm));
4779   PetscCall(DMCreateLocalVector(cdm, &coordinates));
4780   PetscCall(VecSetBlockSize(coordinates, spaceDim));
4781   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4782   PetscCall(VecGetArrayWrite(coordinates, &coords));
4783   for (v = 0; v < vEnd - vStart; ++v) {
4784     for (d = 0; d < spaceDim; ++d) coords[v * spaceDim + d] = vertexCoords[v * spaceDim + d];
4785   }
4786   PetscCall(VecRestoreArrayWrite(coordinates, &coords));
4787   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4788   PetscCall(VecDestroy(&coordinates));
4789   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4790   PetscFunctionReturn(0);
4791 }
4792 
4793 /*@
4794   DMPlexCreateFromCellListPetsc - Create `DMPLEX` from a list of vertices for each cell (common mesh generator output), but only process 0 takes in the input
4795 
4796   Collective
4797 
4798   Input Parameters:
4799 + comm - The communicator
4800 . dim - The topological dimension of the mesh
4801 . numCells - The number of cells, only on process 0
4802 . numVertices - The number of vertices owned by this process, or `PETSC_DECIDE`, only on process 0
4803 . numCorners - The number of vertices for each cell, only on process 0
4804 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4805 . cells - An array of numCells*numCorners numbers, the vertices for each cell, only on process 0
4806 . spaceDim - The spatial dimension used for coordinates
4807 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex, only on process 0
4808 
4809   Output Parameter:
4810 . dm - The `DM`, which only has points on process 0
4811 
4812   Level: intermediate
4813 
4814   Notes:
4815   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`, `DMPlexBuildFromCellList()`,
4816   `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellList()`
4817 
4818   See `DMPlexBuildFromCellList()` for an example and details about the topology-related parameters.
4819   See `DMPlexBuildCoordinatesFromCellList()` for details about the geometry-related parameters.
4820   See `DMPlexCreateFromCellListParallelPetsc()` for parallel input
4821 
4822 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellList()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
4823 @*/
4824 PetscErrorCode DMPlexCreateFromCellListPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], DM *dm)
4825 {
4826   PetscMPIInt rank;
4827 
4828   PetscFunctionBegin;
4829   PetscCheck(dim, comm, PETSC_ERR_ARG_OUTOFRANGE, "This is not appropriate for 0-dimensional meshes. Consider either creating the DM using DMPlexCreateFromDAG(), by hand, or using DMSwarm.");
4830   PetscCallMPI(MPI_Comm_rank(comm, &rank));
4831   PetscCall(DMCreate(comm, dm));
4832   PetscCall(DMSetType(*dm, DMPLEX));
4833   PetscCall(DMSetDimension(*dm, dim));
4834   if (rank == 0) PetscCall(DMPlexBuildFromCellList(*dm, numCells, numVertices, numCorners, cells));
4835   else PetscCall(DMPlexBuildFromCellList(*dm, 0, 0, 0, NULL));
4836   if (interpolate) {
4837     DM idm;
4838 
4839     PetscCall(DMPlexInterpolate(*dm, &idm));
4840     PetscCall(DMDestroy(dm));
4841     *dm = idm;
4842   }
4843   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, vertexCoords));
4844   else PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, NULL));
4845   PetscFunctionReturn(0);
4846 }
4847 
4848 /*@
4849   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a DM
4850 
4851   Input Parameters:
4852 + dm - The empty DM object, usually from DMCreate() and DMSetDimension()
4853 . depth - The depth of the DAG
4854 . numPoints - Array of size depth + 1 containing the number of points at each depth
4855 . coneSize - The cone size of each point
4856 . cones - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
4857 . coneOrientations - The orientation of each cone point
4858 - vertexCoords - An array of numPoints[0]*spacedim numbers representing the coordinates of each vertex, with spacedim the value set via DMSetCoordinateDim()
4859 
4860   Output Parameter:
4861 . dm - The DM
4862 
4863   Note:
4864   Two triangles sharing a face would have input
4865 .vb
4866   depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
4867   cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
4868  vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
4869 .ve
4870 which would result in the DMPlex
4871 .vb
4872         4
4873       / | \
4874      /  |  \
4875     /   |   \
4876    2  0 | 1  5
4877     \   |   /
4878      \  |  /
4879       \ | /
4880         3
4881 .ve
4882  Notice that all points are numbered consecutively, unlike `DMPlexCreateFromCellListPetsc()`
4883 
4884   Level: advanced
4885 
4886 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
4887 @*/
4888 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
4889 {
4890   Vec          coordinates;
4891   PetscSection coordSection;
4892   PetscScalar *coords;
4893   PetscInt     coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
4894 
4895   PetscFunctionBegin;
4896   PetscCall(DMGetDimension(dm, &dim));
4897   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
4898   PetscCheck(dimEmbed >= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Embedding dimension %" PetscInt_FMT " cannot be less than intrinsic dimension %" PetscInt_FMT, dimEmbed, dim);
4899   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
4900   PetscCall(DMPlexSetChart(dm, pStart, pEnd));
4901   for (p = pStart; p < pEnd; ++p) {
4902     PetscCall(DMPlexSetConeSize(dm, p, coneSize[p - pStart]));
4903     if (firstVertex < 0 && !coneSize[p - pStart]) firstVertex = p - pStart;
4904   }
4905   PetscCheck(firstVertex >= 0 || !numPoints[0], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Expected %" PetscInt_FMT " vertices but could not find any", numPoints[0]);
4906   PetscCall(DMSetUp(dm)); /* Allocate space for cones */
4907   for (p = pStart, off = 0; p < pEnd; off += coneSize[p - pStart], ++p) {
4908     PetscCall(DMPlexSetCone(dm, p, &cones[off]));
4909     PetscCall(DMPlexSetConeOrientation(dm, p, &coneOrientations[off]));
4910   }
4911   PetscCall(DMPlexSymmetrize(dm));
4912   PetscCall(DMPlexStratify(dm));
4913   /* Build coordinates */
4914   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4915   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4916   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dimEmbed));
4917   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numPoints[0]));
4918   for (v = firstVertex; v < firstVertex + numPoints[0]; ++v) {
4919     PetscCall(PetscSectionSetDof(coordSection, v, dimEmbed));
4920     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed));
4921   }
4922   PetscCall(PetscSectionSetUp(coordSection));
4923   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4924   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
4925   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4926   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4927   PetscCall(VecSetBlockSize(coordinates, dimEmbed));
4928   PetscCall(VecSetType(coordinates, VECSTANDARD));
4929   if (vertexCoords) {
4930     PetscCall(VecGetArray(coordinates, &coords));
4931     for (v = 0; v < numPoints[0]; ++v) {
4932       PetscInt off;
4933 
4934       PetscCall(PetscSectionGetOffset(coordSection, v + firstVertex, &off));
4935       for (d = 0; d < dimEmbed; ++d) coords[off + d] = vertexCoords[v * dimEmbed + d];
4936     }
4937   }
4938   PetscCall(VecRestoreArray(coordinates, &coords));
4939   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4940   PetscCall(VecDestroy(&coordinates));
4941   PetscFunctionReturn(0);
4942 }
4943 
4944 /*@C
4945   DMPlexCreateCellVertexFromFile - Create a `DMPLEX` mesh from a simple cell-vertex file.
4946 
4947   Collective
4948 
4949 + comm        - The MPI communicator
4950 . filename    - Name of the .dat file
4951 - interpolate - Create faces and edges in the mesh
4952 
4953   Output Parameter:
4954 . dm  - The `DM` object representing the mesh
4955 
4956   Level: beginner
4957 
4958   Note:
4959   The format is the simplest possible:
4960 .vb
4961   Ne
4962   v0 v1 ... vk
4963   Nv
4964   x y z marker
4965 .ve
4966 
4967   Developer Note:
4968   Should use a `PetscViewer` not a filename
4969 
4970 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromFile()`, `DMPlexCreateMedFromFile()`, `DMPlexCreateGmsh()`, `DMPlexCreate()`
4971 @*/
4972 PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
4973 {
4974   DMLabel      marker;
4975   PetscViewer  viewer;
4976   Vec          coordinates;
4977   PetscSection coordSection;
4978   PetscScalar *coords;
4979   char         line[PETSC_MAX_PATH_LEN];
4980   PetscInt     dim = 3, cdim = 3, coordSize, v, c, d;
4981   PetscMPIInt  rank;
4982   int          snum, Nv, Nc, Ncn, Nl;
4983 
4984   PetscFunctionBegin;
4985   PetscCallMPI(MPI_Comm_rank(comm, &rank));
4986   PetscCall(PetscViewerCreate(comm, &viewer));
4987   PetscCall(PetscViewerSetType(viewer, PETSCVIEWERASCII));
4988   PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
4989   PetscCall(PetscViewerFileSetName(viewer, filename));
4990   if (rank == 0) {
4991     PetscCall(PetscViewerRead(viewer, line, 4, NULL, PETSC_STRING));
4992     snum = sscanf(line, "%d %d %d %d", &Nc, &Nv, &Ncn, &Nl);
4993     PetscCheck(snum == 4, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4994   } else {
4995     Nc = Nv = Ncn = Nl = 0;
4996   }
4997   PetscCall(DMCreate(comm, dm));
4998   PetscCall(DMSetType(*dm, DMPLEX));
4999   PetscCall(DMPlexSetChart(*dm, 0, Nc + Nv));
5000   PetscCall(DMSetDimension(*dm, dim));
5001   PetscCall(DMSetCoordinateDim(*dm, cdim));
5002   /* Read topology */
5003   if (rank == 0) {
5004     char     format[PETSC_MAX_PATH_LEN];
5005     PetscInt cone[8];
5006     int      vbuf[8], v;
5007 
5008     for (c = 0; c < Ncn; ++c) {
5009       format[c * 3 + 0] = '%';
5010       format[c * 3 + 1] = 'd';
5011       format[c * 3 + 2] = ' ';
5012     }
5013     format[Ncn * 3 - 1] = '\0';
5014     for (c = 0; c < Nc; ++c) PetscCall(DMPlexSetConeSize(*dm, c, Ncn));
5015     PetscCall(DMSetUp(*dm));
5016     for (c = 0; c < Nc; ++c) {
5017       PetscCall(PetscViewerRead(viewer, line, Ncn, NULL, PETSC_STRING));
5018       switch (Ncn) {
5019       case 2:
5020         snum = sscanf(line, format, &vbuf[0], &vbuf[1]);
5021         break;
5022       case 3:
5023         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2]);
5024         break;
5025       case 4:
5026         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3]);
5027         break;
5028       case 6:
5029         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5]);
5030         break;
5031       case 8:
5032         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);
5033         break;
5034       default:
5035         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No cell shape with %d vertices", Ncn);
5036       }
5037       PetscCheck(snum == Ncn, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5038       for (v = 0; v < Ncn; ++v) cone[v] = vbuf[v] + Nc;
5039       /* Hexahedra are inverted */
5040       if (Ncn == 8) {
5041         PetscInt tmp = cone[1];
5042         cone[1]      = cone[3];
5043         cone[3]      = tmp;
5044       }
5045       PetscCall(DMPlexSetCone(*dm, c, cone));
5046     }
5047   }
5048   PetscCall(DMPlexSymmetrize(*dm));
5049   PetscCall(DMPlexStratify(*dm));
5050   /* Read coordinates */
5051   PetscCall(DMGetCoordinateSection(*dm, &coordSection));
5052   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5053   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
5054   PetscCall(PetscSectionSetChart(coordSection, Nc, Nc + Nv));
5055   for (v = Nc; v < Nc + Nv; ++v) {
5056     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
5057     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
5058   }
5059   PetscCall(PetscSectionSetUp(coordSection));
5060   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
5061   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
5062   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5063   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
5064   PetscCall(VecSetBlockSize(coordinates, cdim));
5065   PetscCall(VecSetType(coordinates, VECSTANDARD));
5066   PetscCall(VecGetArray(coordinates, &coords));
5067   if (rank == 0) {
5068     char   format[PETSC_MAX_PATH_LEN];
5069     double x[3];
5070     int    l, val[3];
5071 
5072     if (Nl) {
5073       for (l = 0; l < Nl; ++l) {
5074         format[l * 3 + 0] = '%';
5075         format[l * 3 + 1] = 'd';
5076         format[l * 3 + 2] = ' ';
5077       }
5078       format[Nl * 3 - 1] = '\0';
5079       PetscCall(DMCreateLabel(*dm, "marker"));
5080       PetscCall(DMGetLabel(*dm, "marker", &marker));
5081     }
5082     for (v = 0; v < Nv; ++v) {
5083       PetscCall(PetscViewerRead(viewer, line, 3 + Nl, NULL, PETSC_STRING));
5084       snum = sscanf(line, "%lg %lg %lg", &x[0], &x[1], &x[2]);
5085       PetscCheck(snum == 3, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5086       switch (Nl) {
5087       case 0:
5088         snum = 0;
5089         break;
5090       case 1:
5091         snum = sscanf(line, format, &val[0]);
5092         break;
5093       case 2:
5094         snum = sscanf(line, format, &val[0], &val[1]);
5095         break;
5096       case 3:
5097         snum = sscanf(line, format, &val[0], &val[1], &val[2]);
5098         break;
5099       default:
5100         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Request support for %d labels", Nl);
5101       }
5102       PetscCheck(snum == Nl, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5103       for (d = 0; d < cdim; ++d) coords[v * cdim + d] = x[d];
5104       for (l = 0; l < Nl; ++l) PetscCall(DMLabelSetValue(marker, v + Nc, val[l]));
5105     }
5106   }
5107   PetscCall(VecRestoreArray(coordinates, &coords));
5108   PetscCall(DMSetCoordinatesLocal(*dm, coordinates));
5109   PetscCall(VecDestroy(&coordinates));
5110   PetscCall(PetscViewerDestroy(&viewer));
5111   if (interpolate) {
5112     DM      idm;
5113     DMLabel bdlabel;
5114 
5115     PetscCall(DMPlexInterpolate(*dm, &idm));
5116     PetscCall(DMDestroy(dm));
5117     *dm = idm;
5118 
5119     if (!Nl) {
5120       PetscCall(DMCreateLabel(*dm, "marker"));
5121       PetscCall(DMGetLabel(*dm, "marker", &bdlabel));
5122       PetscCall(DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel));
5123       PetscCall(DMPlexLabelComplete(*dm, bdlabel));
5124     }
5125   }
5126   PetscFunctionReturn(0);
5127 }
5128 
5129 /*@C
5130   DMPlexCreateFromFile - This takes a filename and produces a `DM`
5131 
5132   Collective
5133 
5134   Input Parameters:
5135 + comm - The communicator
5136 . filename - A file name
5137 . plexname - The object name of the resulting `DM`, also used for intra-datafile lookup by some formats
5138 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
5139 
5140   Output Parameter:
5141 . dm - The `DM`
5142 
5143   Options Database Key:
5144 . -dm_plex_create_from_hdf5_xdmf - use the `PETSC_VIEWER_HDF5_XDMF` format for reading HDF5
5145 
5146   Use -dm_plex_create_ prefix to pass options to the internal PetscViewer, e.g.
5147 $ -dm_plex_create_viewer_hdf5_collective
5148 
5149   Level: beginner
5150 
5151   Notes:
5152   Using `PETSCVIEWERHDF5` type with `PETSC_VIEWER_HDF5_PETSC` format, one can save multiple `DMPLEX`
5153   meshes in a single HDF5 file. This in turn requires one to name the `DMPLEX` object with `PetscObjectSetName()`
5154   before saving it with `DMView()` and before loading it with `DMLoad()` for identification of the mesh object.
5155   The input parameter name is thus used to name the `DMPLEX` object when `DMPlexCreateFromFile()` internally
5156   calls `DMLoad()`. Currently, name is ignored for other viewer types and/or formats.
5157 
5158 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`, `PetscObjectSetName()`, `DMView()`, `DMLoad()`
5159 @*/
5160 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], const char plexname[], PetscBool interpolate, DM *dm)
5161 {
5162   const char  extGmsh[]      = ".msh";
5163   const char  extGmsh2[]     = ".msh2";
5164   const char  extGmsh4[]     = ".msh4";
5165   const char  extCGNS[]      = ".cgns";
5166   const char  extExodus[]    = ".exo";
5167   const char  extExodus_e[]  = ".e";
5168   const char  extGenesis[]   = ".gen";
5169   const char  extFluent[]    = ".cas";
5170   const char  extHDF5[]      = ".h5";
5171   const char  extMed[]       = ".med";
5172   const char  extPLY[]       = ".ply";
5173   const char  extEGADSLite[] = ".egadslite";
5174   const char  extEGADS[]     = ".egads";
5175   const char  extIGES[]      = ".igs";
5176   const char  extSTEP[]      = ".stp";
5177   const char  extCV[]        = ".dat";
5178   size_t      len;
5179   PetscBool   isGmsh, isGmsh2, isGmsh4, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isMed, isPLY, isEGADSLite, isEGADS, isIGES, isSTEP, isCV;
5180   PetscMPIInt rank;
5181 
5182   PetscFunctionBegin;
5183   PetscValidCharPointer(filename, 2);
5184   if (plexname) PetscValidCharPointer(plexname, 3);
5185   PetscValidPointer(dm, 5);
5186   PetscCall(DMInitializePackage());
5187   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5188   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5189   PetscCall(PetscStrlen(filename, &len));
5190   PetscCheck(len, comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
5191 
5192 #define CheckExtension(extension__, is_extension__) \
5193   do { \
5194     PetscAssert(sizeof(extension__), comm, PETSC_ERR_PLIB, "Zero-size extension: %s", extension__); \
5195     /* don't count the null-terminator at the end */ \
5196     const size_t ext_len = sizeof(extension__) - 1; \
5197     if (len < ext_len) { \
5198       is_extension__ = PETSC_FALSE; \
5199     } else { \
5200       PetscCall(PetscStrncmp(filename + len - ext_len, extension__, ext_len, &is_extension__)); \
5201     } \
5202   } while (0)
5203 
5204   CheckExtension(extGmsh, isGmsh);
5205   CheckExtension(extGmsh2, isGmsh2);
5206   CheckExtension(extGmsh4, isGmsh4);
5207   CheckExtension(extCGNS, isCGNS);
5208   CheckExtension(extExodus, isExodus);
5209   if (!isExodus) CheckExtension(extExodus_e, isExodus);
5210   CheckExtension(extGenesis, isGenesis);
5211   CheckExtension(extFluent, isFluent);
5212   CheckExtension(extHDF5, isHDF5);
5213   CheckExtension(extMed, isMed);
5214   CheckExtension(extPLY, isPLY);
5215   CheckExtension(extEGADSLite, isEGADSLite);
5216   CheckExtension(extEGADS, isEGADS);
5217   CheckExtension(extIGES, isIGES);
5218   CheckExtension(extSTEP, isSTEP);
5219   CheckExtension(extCV, isCV);
5220 
5221 #undef CheckExtension
5222 
5223   if (isGmsh || isGmsh2 || isGmsh4) {
5224     PetscCall(DMPlexCreateGmshFromFile(comm, filename, interpolate, dm));
5225   } else if (isCGNS) {
5226     PetscCall(DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm));
5227   } else if (isExodus || isGenesis) {
5228     PetscCall(DMPlexCreateExodusFromFile(comm, filename, interpolate, dm));
5229   } else if (isFluent) {
5230     PetscCall(DMPlexCreateFluentFromFile(comm, filename, interpolate, dm));
5231   } else if (isHDF5) {
5232     PetscBool   load_hdf5_xdmf = PETSC_FALSE;
5233     PetscViewer viewer;
5234 
5235     /* PETSC_VIEWER_HDF5_XDMF is used if the filename ends with .xdmf.h5, or if -dm_plex_create_from_hdf5_xdmf option is present */
5236     PetscCall(PetscStrncmp(&filename[PetscMax(0, len - 8)], ".xdmf", 5, &load_hdf5_xdmf));
5237     PetscCall(PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &load_hdf5_xdmf, NULL));
5238     PetscCall(PetscViewerCreate(comm, &viewer));
5239     PetscCall(PetscViewerSetType(viewer, PETSCVIEWERHDF5));
5240     PetscCall(PetscViewerSetOptionsPrefix(viewer, "dm_plex_create_"));
5241     PetscCall(PetscViewerSetFromOptions(viewer));
5242     PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
5243     PetscCall(PetscViewerFileSetName(viewer, filename));
5244 
5245     PetscCall(DMCreate(comm, dm));
5246     PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5247     PetscCall(DMSetType(*dm, DMPLEX));
5248     if (load_hdf5_xdmf) PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF));
5249     PetscCall(DMLoad(*dm, viewer));
5250     if (load_hdf5_xdmf) PetscCall(PetscViewerPopFormat(viewer));
5251     PetscCall(PetscViewerDestroy(&viewer));
5252 
5253     if (interpolate) {
5254       DM idm;
5255 
5256       PetscCall(DMPlexInterpolate(*dm, &idm));
5257       PetscCall(DMDestroy(dm));
5258       *dm = idm;
5259     }
5260   } else if (isMed) {
5261     PetscCall(DMPlexCreateMedFromFile(comm, filename, interpolate, dm));
5262   } else if (isPLY) {
5263     PetscCall(DMPlexCreatePLYFromFile(comm, filename, interpolate, dm));
5264   } else if (isEGADSLite || isEGADS || isIGES || isSTEP) {
5265     if (isEGADSLite) PetscCall(DMPlexCreateEGADSLiteFromFile(comm, filename, dm));
5266     else PetscCall(DMPlexCreateEGADSFromFile(comm, filename, dm));
5267     if (!interpolate) {
5268       DM udm;
5269 
5270       PetscCall(DMPlexUninterpolate(*dm, &udm));
5271       PetscCall(DMDestroy(dm));
5272       *dm = udm;
5273     }
5274   } else if (isCV) {
5275     PetscCall(DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm));
5276   } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
5277   PetscCall(PetscStrlen(plexname, &len));
5278   if (len) PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5279   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5280   PetscFunctionReturn(0);
5281 }
5282 /*@C
5283   DMPlexCreateEphemeral - This takes a `DMPlexTransform` and a base `DMPlex` and produces an ephemeral `DM`, meaning one that is created on the fly in response to queries.
5284 
5285   Input Parameter:
5286 . tr - The `DMPlexTransform`
5287 
5288   Output Parameter:
5289 . dm - The `DM`
5290 
5291   Notes:
5292   An emphemeral mesh is one that is not stored concretely, as in the default Plex implementation, but rather is produced on the fly in response to queries, using information from the transform and the base mesh.
5293 
5294   Level: beginner
5295 
5296 .seealso: `DMPlexCreateFromFile`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
5297 @*/
5298 PetscErrorCode DMPlexCreateEphemeral(DMPlexTransform tr, DM *dm)
5299 {
5300   DM       bdm;
5301   PetscInt Nl;
5302 
5303   PetscFunctionBegin;
5304   PetscCall(DMCreate(PetscObjectComm((PetscObject)tr), dm));
5305   PetscCall(DMSetType(*dm, DMPLEX));
5306   PetscCall(DMSetFromOptions(*dm));
5307 
5308   PetscCall(PetscObjectReference((PetscObject)tr));
5309   PetscCall(DMPlexTransformDestroy(&((DM_Plex *)(*dm)->data)->tr));
5310   ((DM_Plex *)(*dm)->data)->tr = tr;
5311 
5312   PetscCall(DMPlexTransformGetDM(tr, &bdm));
5313   PetscCall(DMGetNumLabels(bdm, &Nl));
5314   for (PetscInt l = 0; l < Nl; ++l) {
5315     DMLabel     label, labelNew;
5316     const char *lname;
5317     PetscBool   isDepth, isCellType;
5318 
5319     PetscCall(DMGetLabelName(bdm, l, &lname));
5320     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
5321     if (isDepth) continue;
5322     PetscCall(PetscStrcmp(lname, "celltype", &isCellType));
5323     if (isCellType) continue;
5324     PetscCall(DMCreateLabel(*dm, lname));
5325     PetscCall(DMGetLabel(bdm, lname, &label));
5326     PetscCall(DMGetLabel(*dm, lname, &labelNew));
5327     PetscCall(DMLabelSetType(labelNew, DMLABELEPHEMERAL));
5328     PetscCall(DMLabelEphemeralSetLabel(labelNew, label));
5329     PetscCall(DMLabelEphemeralSetTransform(labelNew, tr));
5330     PetscCall(DMLabelSetUp(labelNew));
5331   }
5332   PetscFunctionReturn(0);
5333 }
5334