xref: /petsc/src/dm/impls/plex/plexcreate.c (revision c1730cc1cd3b6fc6205d70c430a9ca69e7268eec)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h> /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>   /*I   "petscdmplex.h"   I*/
4 #include <petscsf.h>
5 #include <petscdmplextransform.h>
6 #include <petscdmlabelephemeral.h>
7 #include <petsc/private/kernels/blockmatmult.h>
8 #include <petsc/private/kernels/blockinvert.h>
9 
10 PetscLogEvent DMPLEX_CreateFromFile, DMPLEX_BuildFromCellList, DMPLEX_BuildCoordinatesFromCellList;
11 
12 /* External function declarations here */
13 static PetscErrorCode DMInitialize_Plex(DM dm);
14 
15 /* This copies internal things in the Plex structure that we generally want when making a new, related Plex */
16 PetscErrorCode DMPlexCopy_Internal(DM dmin, PetscBool copyPeriodicity, PetscBool copyOverlap, DM dmout)
17 {
18   const PetscReal         *maxCell, *Lstart, *L;
19   PetscBool                dist;
20   DMPlexReorderDefaultFlag reorder;
21 
22   PetscFunctionBegin;
23   if (copyPeriodicity) {
24     PetscCall(DMGetPeriodicity(dmin, &maxCell, &Lstart, &L));
25     PetscCall(DMSetPeriodicity(dmout, maxCell, Lstart, L));
26   }
27   PetscCall(DMPlexDistributeGetDefault(dmin, &dist));
28   PetscCall(DMPlexDistributeSetDefault(dmout, dist));
29   PetscCall(DMPlexReorderGetDefault(dmin, &reorder));
30   PetscCall(DMPlexReorderSetDefault(dmout, reorder));
31   ((DM_Plex *)dmout->data)->useHashLocation = ((DM_Plex *)dmin->data)->useHashLocation;
32   if (copyOverlap) PetscCall(DMPlexSetOverlap_Plex(dmout, dmin, 0));
33   PetscFunctionReturn(0);
34 }
35 
36 /* Replace dm with the contents of ndm, and then destroy ndm
37    - Share the DM_Plex structure
38    - Share the coordinates
39    - Share the SF
40 */
41 PetscErrorCode DMPlexReplace_Internal(DM dm, DM *ndm)
42 {
43   PetscSF          sf;
44   DM               dmNew = *ndm, coordDM, coarseDM;
45   Vec              coords;
46   const PetscReal *maxCell, *Lstart, *L;
47   PetscInt         dim, cdim;
48 
49   PetscFunctionBegin;
50   if (dm == dmNew) {
51     PetscCall(DMDestroy(ndm));
52     PetscFunctionReturn(0);
53   }
54   dm->setupcalled = dmNew->setupcalled;
55   PetscCall(DMGetDimension(dmNew, &dim));
56   PetscCall(DMSetDimension(dm, dim));
57   PetscCall(DMGetCoordinateDim(dmNew, &cdim));
58   PetscCall(DMSetCoordinateDim(dm, cdim));
59   PetscCall(DMGetPointSF(dmNew, &sf));
60   PetscCall(DMSetPointSF(dm, sf));
61   PetscCall(DMGetCoordinateDM(dmNew, &coordDM));
62   PetscCall(DMGetCoordinatesLocal(dmNew, &coords));
63   PetscCall(DMSetCoordinateDM(dm, coordDM));
64   PetscCall(DMSetCoordinatesLocal(dm, coords));
65   PetscCall(DMGetCellCoordinateDM(dmNew, &coordDM));
66   PetscCall(DMGetCellCoordinatesLocal(dmNew, &coords));
67   PetscCall(DMSetCellCoordinateDM(dm, coordDM));
68   PetscCall(DMSetCellCoordinatesLocal(dm, coords));
69   /* Do not want to create the coordinate field if it does not already exist, so do not call DMGetCoordinateField() */
70   PetscCall(DMFieldDestroy(&dm->coordinates[0].field));
71   dm->coordinates[0].field            = dmNew->coordinates[0].field;
72   ((DM_Plex *)dmNew->data)->coordFunc = ((DM_Plex *)dm->data)->coordFunc;
73   PetscCall(DMGetPeriodicity(dmNew, &maxCell, &Lstart, &L));
74   PetscCall(DMSetPeriodicity(dm, maxCell, Lstart, L));
75   PetscCall(DMDestroy_Plex(dm));
76   PetscCall(DMInitialize_Plex(dm));
77   dm->data = dmNew->data;
78   ((DM_Plex *)dmNew->data)->refct++;
79   PetscCall(DMDestroyLabelLinkList_Internal(dm));
80   PetscCall(DMCopyLabels(dmNew, dm, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL));
81   PetscCall(DMGetCoarseDM(dmNew, &coarseDM));
82   PetscCall(DMSetCoarseDM(dm, coarseDM));
83   PetscCall(DMDestroy(ndm));
84   PetscFunctionReturn(0);
85 }
86 
87 /* Swap dm with the contents of dmNew
88    - Swap the DM_Plex structure
89    - Swap the coordinates
90    - Swap the point PetscSF
91 */
92 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
93 {
94   DM          coordDMA, coordDMB;
95   Vec         coordsA, coordsB;
96   PetscSF     sfA, sfB;
97   DMField     fieldTmp;
98   void       *tmp;
99   DMLabelLink listTmp;
100   DMLabel     depthTmp;
101   PetscInt    tmpI;
102 
103   PetscFunctionBegin;
104   if (dmA == dmB) PetscFunctionReturn(0);
105   PetscCall(DMGetPointSF(dmA, &sfA));
106   PetscCall(DMGetPointSF(dmB, &sfB));
107   PetscCall(PetscObjectReference((PetscObject)sfA));
108   PetscCall(DMSetPointSF(dmA, sfB));
109   PetscCall(DMSetPointSF(dmB, sfA));
110   PetscCall(PetscObjectDereference((PetscObject)sfA));
111 
112   PetscCall(DMGetCoordinateDM(dmA, &coordDMA));
113   PetscCall(DMGetCoordinateDM(dmB, &coordDMB));
114   PetscCall(PetscObjectReference((PetscObject)coordDMA));
115   PetscCall(DMSetCoordinateDM(dmA, coordDMB));
116   PetscCall(DMSetCoordinateDM(dmB, coordDMA));
117   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
118 
119   PetscCall(DMGetCoordinatesLocal(dmA, &coordsA));
120   PetscCall(DMGetCoordinatesLocal(dmB, &coordsB));
121   PetscCall(PetscObjectReference((PetscObject)coordsA));
122   PetscCall(DMSetCoordinatesLocal(dmA, coordsB));
123   PetscCall(DMSetCoordinatesLocal(dmB, coordsA));
124   PetscCall(PetscObjectDereference((PetscObject)coordsA));
125 
126   PetscCall(DMGetCellCoordinateDM(dmA, &coordDMA));
127   PetscCall(DMGetCellCoordinateDM(dmB, &coordDMB));
128   PetscCall(PetscObjectReference((PetscObject)coordDMA));
129   PetscCall(DMSetCellCoordinateDM(dmA, coordDMB));
130   PetscCall(DMSetCellCoordinateDM(dmB, coordDMA));
131   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
132 
133   PetscCall(DMGetCellCoordinatesLocal(dmA, &coordsA));
134   PetscCall(DMGetCellCoordinatesLocal(dmB, &coordsB));
135   PetscCall(PetscObjectReference((PetscObject)coordsA));
136   PetscCall(DMSetCellCoordinatesLocal(dmA, coordsB));
137   PetscCall(DMSetCellCoordinatesLocal(dmB, coordsA));
138   PetscCall(PetscObjectDereference((PetscObject)coordsA));
139 
140   fieldTmp                  = dmA->coordinates[0].field;
141   dmA->coordinates[0].field = dmB->coordinates[0].field;
142   dmB->coordinates[0].field = fieldTmp;
143   fieldTmp                  = dmA->coordinates[1].field;
144   dmA->coordinates[1].field = dmB->coordinates[1].field;
145   dmB->coordinates[1].field = fieldTmp;
146   tmp                       = dmA->data;
147   dmA->data                 = dmB->data;
148   dmB->data                 = tmp;
149   listTmp                   = dmA->labels;
150   dmA->labels               = dmB->labels;
151   dmB->labels               = listTmp;
152   depthTmp                  = dmA->depthLabel;
153   dmA->depthLabel           = dmB->depthLabel;
154   dmB->depthLabel           = depthTmp;
155   depthTmp                  = dmA->celltypeLabel;
156   dmA->celltypeLabel        = dmB->celltypeLabel;
157   dmB->celltypeLabel        = depthTmp;
158   tmpI                      = dmA->levelup;
159   dmA->levelup              = dmB->levelup;
160   dmB->levelup              = tmpI;
161   PetscFunctionReturn(0);
162 }
163 
164 static PetscErrorCode DMPlexInterpolateInPlace_Internal(DM dm)
165 {
166   DM idm;
167 
168   PetscFunctionBegin;
169   PetscCall(DMPlexInterpolate(dm, &idm));
170   PetscCall(DMPlexCopyCoordinates(dm, idm));
171   PetscCall(DMPlexReplace_Internal(dm, &idm));
172   PetscFunctionReturn(0);
173 }
174 
175 /*@C
176   DMPlexCreateCoordinateSpace - Creates a finite element space for the coordinates
177 
178   Collective
179 
180   Input Parameters:
181 + DM        - The DM
182 . degree    - The degree of the finite element or PETSC_DECIDE
183 - coordFunc - An optional function to map new points from refinement to the surface
184 
185   Level: advanced
186 
187 .seealso: `PetscFECreateLagrange()`, `DMGetCoordinateDM()`
188 @*/
189 PetscErrorCode DMPlexCreateCoordinateSpace(DM dm, PetscInt degree, PetscPointFunc coordFunc)
190 {
191   DM_Plex     *mesh = (DM_Plex *)dm->data;
192   DM           cdm;
193   PetscDS      cds;
194   PetscFE      fe;
195   PetscClassId id;
196 
197   PetscFunctionBegin;
198   PetscCall(DMGetCoordinateDM(dm, &cdm));
199   PetscCall(DMGetDS(cdm, &cds));
200   PetscCall(PetscDSGetDiscretization(cds, 0, (PetscObject *)&fe));
201   PetscCall(PetscObjectGetClassId((PetscObject)fe, &id));
202   if (id != PETSCFE_CLASSID) {
203     PetscInt dim, dE, qorder;
204 
205     PetscCall(DMGetDimension(dm, &dim));
206     PetscCall(DMGetCoordinateDim(dm, &dE));
207     qorder = degree;
208     PetscObjectOptionsBegin((PetscObject)cdm);
209     PetscCall(PetscOptionsBoundedInt("-default_quadrature_order", "Quadrature order is one less than quadrature points per edge", "DMPlexCreateCoordinateSpace", qorder, &qorder, NULL, 0));
210     PetscOptionsEnd();
211     if (degree == PETSC_DECIDE) fe = NULL;
212     else {
213       DMPolytopeType ct;
214       PetscInt       cStart, cEnd, ctTmp;
215 
216       PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
217       // Want to match cell types
218       if (cEnd > cStart) PetscCall(DMPlexGetCellType(dm, cStart, &ct));
219       else ct = DM_POLYTOPE_UNKNOWN;
220       ctTmp = (PetscInt)ct;
221       PetscCallMPI(MPI_Allreduce(MPI_IN_PLACE, &ctTmp, 1, MPIU_INT, MPI_MIN, PetscObjectComm((PetscObject)dm)));
222       ct = (DMPolytopeType)ctTmp;
223       // Work around current bug
224       if (ct == DM_POLYTOPE_SEG_PRISM_TENSOR || ct == DM_POLYTOPE_TRI_PRISM_TENSOR || ct == DM_POLYTOPE_QUAD_PRISM_TENSOR) fe = NULL;
225       else PetscCall(PetscFECreateLagrangeByCell(PETSC_COMM_SELF, dim, dE, ct, degree, qorder, &fe));
226     }
227     if (fe) PetscCall(DMProjectCoordinates(dm, fe));
228     PetscCall(PetscFEDestroy(&fe));
229   }
230   mesh->coordFunc = coordFunc;
231   PetscFunctionReturn(0);
232 }
233 
234 /*@
235   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
236 
237   Collective
238 
239   Input Parameters:
240 + comm - The communicator for the DM object
241 . dim - The spatial dimension
242 . simplex - Flag for simplicial cells, otherwise they are tensor product cells
243 . interpolate - Flag to create intermediate mesh pieces (edges, faces)
244 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
245 
246   Output Parameter:
247 . dm - The DM object
248 
249   Level: beginner
250 
251 .seealso: `DMSetType()`, `DMCreate()`
252 @*/
253 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscReal refinementLimit, DM *newdm)
254 {
255   DM          dm;
256   PetscMPIInt rank;
257 
258   PetscFunctionBegin;
259   PetscCall(DMCreate(comm, &dm));
260   PetscCall(DMSetType(dm, DMPLEX));
261   PetscCall(DMSetDimension(dm, dim));
262   PetscCallMPI(MPI_Comm_rank(comm, &rank));
263   switch (dim) {
264   case 2:
265     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "triangular"));
266     else PetscCall(PetscObjectSetName((PetscObject)dm, "quadrilateral"));
267     break;
268   case 3:
269     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "tetrahedral"));
270     else PetscCall(PetscObjectSetName((PetscObject)dm, "hexahedral"));
271     break;
272   default:
273     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
274   }
275   if (rank) {
276     PetscInt numPoints[2] = {0, 0};
277     PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL));
278   } else {
279     switch (dim) {
280     case 2:
281       if (simplex) {
282         PetscInt    numPoints[2]        = {4, 2};
283         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
284         PetscInt    cones[6]            = {2, 3, 4, 5, 4, 3};
285         PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
286         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
287 
288         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
289       } else {
290         PetscInt    numPoints[2]        = {6, 2};
291         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
292         PetscInt    cones[8]            = {2, 3, 4, 5, 3, 6, 7, 4};
293         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
294         PetscScalar vertexCoords[12]    = {-1.0, -0.5, 0.0, -0.5, 0.0, 0.5, -1.0, 0.5, 1.0, -0.5, 1.0, 0.5};
295 
296         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
297       }
298       break;
299     case 3:
300       if (simplex) {
301         PetscInt    numPoints[2]        = {5, 2};
302         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
303         PetscInt    cones[8]            = {4, 3, 5, 2, 5, 3, 4, 6};
304         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
305         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0};
306 
307         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
308       } else {
309         PetscInt    numPoints[2]         = {12, 2};
310         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
311         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9, 5, 4, 10, 11, 7, 12, 13, 8};
312         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
313         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5, -1.0, 0.5, -0.5, 0.0, 0.5, -0.5, 0.0, -0.5, -0.5, -1.0, -0.5, 0.5, 0.0, -0.5, 0.5, 0.0, 0.5, 0.5, -1.0, 0.5, 0.5, 1.0, 0.5, -0.5, 1.0, -0.5, -0.5, 1.0, -0.5, 0.5, 1.0, 0.5, 0.5};
314 
315         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
316       }
317       break;
318     default:
319       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
320     }
321   }
322   *newdm = dm;
323   if (refinementLimit > 0.0) {
324     DM          rdm;
325     const char *name;
326 
327     PetscCall(DMPlexSetRefinementUniform(*newdm, PETSC_FALSE));
328     PetscCall(DMPlexSetRefinementLimit(*newdm, refinementLimit));
329     PetscCall(DMRefine(*newdm, comm, &rdm));
330     PetscCall(PetscObjectGetName((PetscObject)*newdm, &name));
331     PetscCall(PetscObjectSetName((PetscObject)rdm, name));
332     PetscCall(DMDestroy(newdm));
333     *newdm = rdm;
334   }
335   if (interpolate) {
336     DM idm;
337 
338     PetscCall(DMPlexInterpolate(*newdm, &idm));
339     PetscCall(DMDestroy(newdm));
340     *newdm = idm;
341   }
342   PetscFunctionReturn(0);
343 }
344 
345 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
346 {
347   const PetscInt numVertices    = 2;
348   PetscInt       markerRight    = 1;
349   PetscInt       markerLeft     = 1;
350   PetscBool      markerSeparate = PETSC_FALSE;
351   Vec            coordinates;
352   PetscSection   coordSection;
353   PetscScalar   *coords;
354   PetscInt       coordSize;
355   PetscMPIInt    rank;
356   PetscInt       cdim = 1, v;
357 
358   PetscFunctionBegin;
359   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
360   if (markerSeparate) {
361     markerRight = 2;
362     markerLeft  = 1;
363   }
364   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
365   if (rank == 0) {
366     PetscCall(DMPlexSetChart(dm, 0, numVertices));
367     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
368     PetscCall(DMSetLabelValue(dm, "marker", 0, markerLeft));
369     PetscCall(DMSetLabelValue(dm, "marker", 1, markerRight));
370   }
371   PetscCall(DMPlexSymmetrize(dm));
372   PetscCall(DMPlexStratify(dm));
373   /* Build coordinates */
374   PetscCall(DMSetCoordinateDim(dm, cdim));
375   PetscCall(DMGetCoordinateSection(dm, &coordSection));
376   PetscCall(PetscSectionSetNumFields(coordSection, 1));
377   PetscCall(PetscSectionSetChart(coordSection, 0, numVertices));
378   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
379   for (v = 0; v < numVertices; ++v) {
380     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
381     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
382   }
383   PetscCall(PetscSectionSetUp(coordSection));
384   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
385   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
386   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
387   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
388   PetscCall(VecSetBlockSize(coordinates, cdim));
389   PetscCall(VecSetType(coordinates, VECSTANDARD));
390   PetscCall(VecGetArray(coordinates, &coords));
391   coords[0] = lower[0];
392   coords[1] = upper[0];
393   PetscCall(VecRestoreArray(coordinates, &coords));
394   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
395   PetscCall(VecDestroy(&coordinates));
396   PetscFunctionReturn(0);
397 }
398 
399 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
400 {
401   const PetscInt numVertices    = (edges[0] + 1) * (edges[1] + 1);
402   const PetscInt numEdges       = edges[0] * (edges[1] + 1) + (edges[0] + 1) * edges[1];
403   PetscInt       markerTop      = 1;
404   PetscInt       markerBottom   = 1;
405   PetscInt       markerRight    = 1;
406   PetscInt       markerLeft     = 1;
407   PetscBool      markerSeparate = PETSC_FALSE;
408   Vec            coordinates;
409   PetscSection   coordSection;
410   PetscScalar   *coords;
411   PetscInt       coordSize;
412   PetscMPIInt    rank;
413   PetscInt       v, vx, vy;
414 
415   PetscFunctionBegin;
416   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
417   if (markerSeparate) {
418     markerTop    = 3;
419     markerBottom = 1;
420     markerRight  = 2;
421     markerLeft   = 4;
422   }
423   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
424   if (rank == 0) {
425     PetscInt e, ex, ey;
426 
427     PetscCall(DMPlexSetChart(dm, 0, numEdges + numVertices));
428     for (e = 0; e < numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
429     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
430     for (vx = 0; vx <= edges[0]; vx++) {
431       for (ey = 0; ey < edges[1]; ey++) {
432         PetscInt edge   = vx * edges[1] + ey + edges[0] * (edges[1] + 1);
433         PetscInt vertex = ey * (edges[0] + 1) + vx + numEdges;
434         PetscInt cone[2];
435 
436         cone[0] = vertex;
437         cone[1] = vertex + edges[0] + 1;
438         PetscCall(DMPlexSetCone(dm, edge, cone));
439         if (vx == edges[0]) {
440           PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
441           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
442           if (ey == edges[1] - 1) {
443             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
444             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerRight));
445           }
446         } else if (vx == 0) {
447           PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
448           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
449           if (ey == edges[1] - 1) {
450             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
451             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft));
452           }
453         }
454       }
455     }
456     for (vy = 0; vy <= edges[1]; vy++) {
457       for (ex = 0; ex < edges[0]; ex++) {
458         PetscInt edge   = vy * edges[0] + ex;
459         PetscInt vertex = vy * (edges[0] + 1) + ex + numEdges;
460         PetscInt cone[2];
461 
462         cone[0] = vertex;
463         cone[1] = vertex + 1;
464         PetscCall(DMPlexSetCone(dm, edge, cone));
465         if (vy == edges[1]) {
466           PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
467           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
468           if (ex == edges[0] - 1) {
469             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
470             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerTop));
471           }
472         } else if (vy == 0) {
473           PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
474           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
475           if (ex == edges[0] - 1) {
476             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
477             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom));
478           }
479         }
480       }
481     }
482   }
483   PetscCall(DMPlexSymmetrize(dm));
484   PetscCall(DMPlexStratify(dm));
485   /* Build coordinates */
486   PetscCall(DMSetCoordinateDim(dm, 2));
487   PetscCall(DMGetCoordinateSection(dm, &coordSection));
488   PetscCall(PetscSectionSetNumFields(coordSection, 1));
489   PetscCall(PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices));
490   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 2));
491   for (v = numEdges; v < numEdges + numVertices; ++v) {
492     PetscCall(PetscSectionSetDof(coordSection, v, 2));
493     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 2));
494   }
495   PetscCall(PetscSectionSetUp(coordSection));
496   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
497   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
498   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
499   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
500   PetscCall(VecSetBlockSize(coordinates, 2));
501   PetscCall(VecSetType(coordinates, VECSTANDARD));
502   PetscCall(VecGetArray(coordinates, &coords));
503   for (vy = 0; vy <= edges[1]; ++vy) {
504     for (vx = 0; vx <= edges[0]; ++vx) {
505       coords[(vy * (edges[0] + 1) + vx) * 2 + 0] = lower[0] + ((upper[0] - lower[0]) / edges[0]) * vx;
506       coords[(vy * (edges[0] + 1) + vx) * 2 + 1] = lower[1] + ((upper[1] - lower[1]) / edges[1]) * vy;
507     }
508   }
509   PetscCall(VecRestoreArray(coordinates, &coords));
510   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
511   PetscCall(VecDestroy(&coordinates));
512   PetscFunctionReturn(0);
513 }
514 
515 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
516 {
517   PetscInt     vertices[3], numVertices;
518   PetscInt     numFaces       = 2 * faces[0] * faces[1] + 2 * faces[1] * faces[2] + 2 * faces[0] * faces[2];
519   PetscInt     markerTop      = 1;
520   PetscInt     markerBottom   = 1;
521   PetscInt     markerFront    = 1;
522   PetscInt     markerBack     = 1;
523   PetscInt     markerRight    = 1;
524   PetscInt     markerLeft     = 1;
525   PetscBool    markerSeparate = PETSC_FALSE;
526   Vec          coordinates;
527   PetscSection coordSection;
528   PetscScalar *coords;
529   PetscInt     coordSize;
530   PetscMPIInt  rank;
531   PetscInt     v, vx, vy, vz;
532   PetscInt     voffset, iface = 0, cone[4];
533 
534   PetscFunctionBegin;
535   PetscCheck(faces[0] >= 1 && faces[1] >= 1 && faces[2] >= 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
536   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
537   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
538   if (markerSeparate) {
539     markerBottom = 1;
540     markerTop    = 2;
541     markerFront  = 3;
542     markerBack   = 4;
543     markerRight  = 5;
544     markerLeft   = 6;
545   }
546   vertices[0] = faces[0] + 1;
547   vertices[1] = faces[1] + 1;
548   vertices[2] = faces[2] + 1;
549   numVertices = vertices[0] * vertices[1] * vertices[2];
550   if (rank == 0) {
551     PetscInt f;
552 
553     PetscCall(DMPlexSetChart(dm, 0, numFaces + numVertices));
554     for (f = 0; f < numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
555     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
556 
557     /* Side 0 (Top) */
558     for (vy = 0; vy < faces[1]; vy++) {
559       for (vx = 0; vx < faces[0]; vx++) {
560         voffset = numFaces + vertices[0] * vertices[1] * (vertices[2] - 1) + vy * vertices[0] + vx;
561         cone[0] = voffset;
562         cone[1] = voffset + 1;
563         cone[2] = voffset + vertices[0] + 1;
564         cone[3] = voffset + vertices[0];
565         PetscCall(DMPlexSetCone(dm, iface, cone));
566         PetscCall(DMSetLabelValue(dm, "marker", iface, markerTop));
567         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerTop));
568         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerTop));
569         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerTop));
570         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerTop));
571         iface++;
572       }
573     }
574 
575     /* Side 1 (Bottom) */
576     for (vy = 0; vy < faces[1]; vy++) {
577       for (vx = 0; vx < faces[0]; vx++) {
578         voffset = numFaces + vy * (faces[0] + 1) + vx;
579         cone[0] = voffset + 1;
580         cone[1] = voffset;
581         cone[2] = voffset + vertices[0];
582         cone[3] = voffset + vertices[0] + 1;
583         PetscCall(DMPlexSetCone(dm, iface, cone));
584         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBottom));
585         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBottom));
586         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBottom));
587         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerBottom));
588         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerBottom));
589         iface++;
590       }
591     }
592 
593     /* Side 2 (Front) */
594     for (vz = 0; vz < faces[2]; vz++) {
595       for (vx = 0; vx < faces[0]; vx++) {
596         voffset = numFaces + vz * vertices[0] * vertices[1] + vx;
597         cone[0] = voffset;
598         cone[1] = voffset + 1;
599         cone[2] = voffset + vertices[0] * vertices[1] + 1;
600         cone[3] = voffset + vertices[0] * vertices[1];
601         PetscCall(DMPlexSetCone(dm, iface, cone));
602         PetscCall(DMSetLabelValue(dm, "marker", iface, markerFront));
603         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerFront));
604         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerFront));
605         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerFront));
606         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerFront));
607         iface++;
608       }
609     }
610 
611     /* Side 3 (Back) */
612     for (vz = 0; vz < faces[2]; vz++) {
613       for (vx = 0; vx < faces[0]; vx++) {
614         voffset = numFaces + vz * vertices[0] * vertices[1] + vertices[0] * (vertices[1] - 1) + vx;
615         cone[0] = voffset + vertices[0] * vertices[1];
616         cone[1] = voffset + vertices[0] * vertices[1] + 1;
617         cone[2] = voffset + 1;
618         cone[3] = voffset;
619         PetscCall(DMPlexSetCone(dm, iface, cone));
620         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBack));
621         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBack));
622         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBack));
623         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerBack));
624         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerBack));
625         iface++;
626       }
627     }
628 
629     /* Side 4 (Left) */
630     for (vz = 0; vz < faces[2]; vz++) {
631       for (vy = 0; vy < faces[1]; vy++) {
632         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0];
633         cone[0] = voffset;
634         cone[1] = voffset + vertices[0] * vertices[1];
635         cone[2] = voffset + vertices[0] * vertices[1] + vertices[0];
636         cone[3] = voffset + vertices[0];
637         PetscCall(DMPlexSetCone(dm, iface, cone));
638         PetscCall(DMSetLabelValue(dm, "marker", iface, markerLeft));
639         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerLeft));
640         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerLeft));
641         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[1] + 0, markerLeft));
642         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerLeft));
643         iface++;
644       }
645     }
646 
647     /* Side 5 (Right) */
648     for (vz = 0; vz < faces[2]; vz++) {
649       for (vy = 0; vy < faces[1]; vy++) {
650         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0] + faces[0];
651         cone[0] = voffset + vertices[0] * vertices[1];
652         cone[1] = voffset;
653         cone[2] = voffset + vertices[0];
654         cone[3] = voffset + vertices[0] * vertices[1] + vertices[0];
655         PetscCall(DMPlexSetCone(dm, iface, cone));
656         PetscCall(DMSetLabelValue(dm, "marker", iface, markerRight));
657         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerRight));
658         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerRight));
659         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerRight));
660         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerRight));
661         iface++;
662       }
663     }
664   }
665   PetscCall(DMPlexSymmetrize(dm));
666   PetscCall(DMPlexStratify(dm));
667   /* Build coordinates */
668   PetscCall(DMSetCoordinateDim(dm, 3));
669   PetscCall(DMGetCoordinateSection(dm, &coordSection));
670   PetscCall(PetscSectionSetNumFields(coordSection, 1));
671   PetscCall(PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices));
672   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 3));
673   for (v = numFaces; v < numFaces + numVertices; ++v) {
674     PetscCall(PetscSectionSetDof(coordSection, v, 3));
675     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 3));
676   }
677   PetscCall(PetscSectionSetUp(coordSection));
678   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
679   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
680   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
681   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
682   PetscCall(VecSetBlockSize(coordinates, 3));
683   PetscCall(VecSetType(coordinates, VECSTANDARD));
684   PetscCall(VecGetArray(coordinates, &coords));
685   for (vz = 0; vz <= faces[2]; ++vz) {
686     for (vy = 0; vy <= faces[1]; ++vy) {
687       for (vx = 0; vx <= faces[0]; ++vx) {
688         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 0] = lower[0] + ((upper[0] - lower[0]) / faces[0]) * vx;
689         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 1] = lower[1] + ((upper[1] - lower[1]) / faces[1]) * vy;
690         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 2] = lower[2] + ((upper[2] - lower[2]) / faces[2]) * vz;
691       }
692     }
693   }
694   PetscCall(VecRestoreArray(coordinates, &coords));
695   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
696   PetscCall(VecDestroy(&coordinates));
697   PetscFunctionReturn(0);
698 }
699 
700 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate)
701 {
702   PetscFunctionBegin;
703   PetscValidLogicalCollectiveInt(dm, dim, 2);
704   PetscCall(DMSetDimension(dm, dim - 1));
705   PetscCall(DMSetCoordinateDim(dm, dim));
706   switch (dim) {
707   case 1:
708     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(dm, lower, upper, faces));
709     break;
710   case 2:
711     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(dm, lower, upper, faces));
712     break;
713   case 3:
714     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(dm, lower, upper, faces));
715     break;
716   default:
717     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Dimension not supported: %" PetscInt_FMT, dim);
718   }
719   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
720   PetscFunctionReturn(0);
721 }
722 
723 /*@C
724   DMPlexCreateBoxSurfaceMesh - Creates a mesh on the surface of the tensor product of unit intervals (box) using tensor cells (hexahedra).
725 
726   Collective
727 
728   Input Parameters:
729 + comm        - The communicator for the DM object
730 . dim         - The spatial dimension of the box, so the resulting mesh is has dimension dim-1
731 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
732 . lower       - The lower left corner, or NULL for (0, 0, 0)
733 . upper       - The upper right corner, or NULL for (1, 1, 1)
734 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
735 
736   Output Parameter:
737 . dm  - The DM object
738 
739   Level: beginner
740 
741 .seealso: `DMSetFromOptions()`, `DMPlexCreateBoxMesh()`, `DMPlexCreateFromFile()`, `DMSetType()`, `DMCreate()`
742 @*/
743 PetscErrorCode DMPlexCreateBoxSurfaceMesh(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate, DM *dm)
744 {
745   PetscInt  fac[3] = {1, 1, 1};
746   PetscReal low[3] = {0, 0, 0};
747   PetscReal upp[3] = {1, 1, 1};
748 
749   PetscFunctionBegin;
750   PetscCall(DMCreate(comm, dm));
751   PetscCall(DMSetType(*dm, DMPLEX));
752   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(*dm, dim, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, interpolate));
753   PetscFunctionReturn(0);
754 }
755 
756 static PetscErrorCode DMPlexCreateLineMesh_Internal(DM dm, PetscInt segments, PetscReal lower, PetscReal upper, DMBoundaryType bd)
757 {
758   PetscInt     i, fStart, fEnd, numCells = 0, numVerts = 0;
759   PetscInt     numPoints[2], *coneSize, *cones, *coneOrientations;
760   PetscScalar *vertexCoords;
761   PetscReal    L, maxCell;
762   PetscBool    markerSeparate = PETSC_FALSE;
763   PetscInt     markerLeft = 1, faceMarkerLeft = 1;
764   PetscInt     markerRight = 1, faceMarkerRight = 2;
765   PetscBool    wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
766   PetscMPIInt  rank;
767 
768   PetscFunctionBegin;
769   PetscValidPointer(dm, 1);
770 
771   PetscCall(DMSetDimension(dm, 1));
772   PetscCall(DMCreateLabel(dm, "marker"));
773   PetscCall(DMCreateLabel(dm, "Face Sets"));
774 
775   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
776   if (rank == 0) numCells = segments;
777   if (rank == 0) numVerts = segments + (wrap ? 0 : 1);
778 
779   numPoints[0] = numVerts;
780   numPoints[1] = numCells;
781   PetscCall(PetscMalloc4(numCells + numVerts, &coneSize, numCells * 2, &cones, numCells + numVerts, &coneOrientations, numVerts, &vertexCoords));
782   PetscCall(PetscArrayzero(coneOrientations, numCells + numVerts));
783   for (i = 0; i < numCells; ++i) coneSize[i] = 2;
784   for (i = 0; i < numVerts; ++i) coneSize[numCells + i] = 0;
785   for (i = 0; i < numCells; ++i) {
786     cones[2 * i]     = numCells + i % numVerts;
787     cones[2 * i + 1] = numCells + (i + 1) % numVerts;
788   }
789   for (i = 0; i < numVerts; ++i) vertexCoords[i] = lower + (upper - lower) * ((PetscReal)i / (PetscReal)numCells);
790   PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
791   PetscCall(PetscFree4(coneSize, cones, coneOrientations, vertexCoords));
792 
793   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
794   if (markerSeparate) {
795     markerLeft  = faceMarkerLeft;
796     markerRight = faceMarkerRight;
797   }
798   if (!wrap && rank == 0) {
799     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
800     PetscCall(DMSetLabelValue(dm, "marker", fStart, markerLeft));
801     PetscCall(DMSetLabelValue(dm, "marker", fEnd - 1, markerRight));
802     PetscCall(DMSetLabelValue(dm, "Face Sets", fStart, faceMarkerLeft));
803     PetscCall(DMSetLabelValue(dm, "Face Sets", fEnd - 1, faceMarkerRight));
804   }
805   if (wrap) {
806     L       = upper - lower;
807     maxCell = (PetscReal)1.1 * (L / (PetscReal)PetscMax(1, segments));
808     PetscCall(DMSetPeriodicity(dm, &maxCell, &lower, &L));
809   }
810   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
811   PetscFunctionReturn(0);
812 }
813 
814 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
815 {
816   DM      boundary, vol;
817   DMLabel bdlabel;
818 
819   PetscFunctionBegin;
820   PetscValidPointer(dm, 1);
821   for (PetscInt i = 0; i < dim; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
822   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &boundary));
823   PetscCall(DMSetType(boundary, DMPLEX));
824   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(boundary, dim, faces, lower, upper, PETSC_FALSE));
825   PetscCall(DMPlexGenerate(boundary, NULL, interpolate, &vol));
826   PetscCall(DMGetLabel(vol, "marker", &bdlabel));
827   if (bdlabel) PetscCall(DMPlexLabelComplete(vol, bdlabel));
828   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_FALSE, vol));
829   PetscCall(DMPlexReplace_Internal(dm, &vol));
830   PetscCall(DMDestroy(&boundary));
831   PetscFunctionReturn(0);
832 }
833 
834 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
835 {
836   DMLabel     cutLabel  = NULL;
837   PetscInt    markerTop = 1, faceMarkerTop = 1;
838   PetscInt    markerBottom = 1, faceMarkerBottom = 1;
839   PetscInt    markerFront = 1, faceMarkerFront = 1;
840   PetscInt    markerBack = 1, faceMarkerBack = 1;
841   PetscInt    markerRight = 1, faceMarkerRight = 1;
842   PetscInt    markerLeft = 1, faceMarkerLeft = 1;
843   PetscInt    dim;
844   PetscBool   markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
845   PetscMPIInt rank;
846 
847   PetscFunctionBegin;
848   PetscCall(DMGetDimension(dm, &dim));
849   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
850   PetscCall(DMCreateLabel(dm, "marker"));
851   PetscCall(DMCreateLabel(dm, "Face Sets"));
852   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
853   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST || bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST || bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
854     if (cutMarker) {
855       PetscCall(DMCreateLabel(dm, "periodic_cut"));
856       PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
857     }
858   }
859   switch (dim) {
860   case 2:
861     faceMarkerTop    = 3;
862     faceMarkerBottom = 1;
863     faceMarkerRight  = 2;
864     faceMarkerLeft   = 4;
865     break;
866   case 3:
867     faceMarkerBottom = 1;
868     faceMarkerTop    = 2;
869     faceMarkerFront  = 3;
870     faceMarkerBack   = 4;
871     faceMarkerRight  = 5;
872     faceMarkerLeft   = 6;
873     break;
874   default:
875     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
876   }
877   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
878   if (markerSeparate) {
879     markerBottom = faceMarkerBottom;
880     markerTop    = faceMarkerTop;
881     markerFront  = faceMarkerFront;
882     markerBack   = faceMarkerBack;
883     markerRight  = faceMarkerRight;
884     markerLeft   = faceMarkerLeft;
885   }
886   {
887     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
888     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
889     const PetscInt numZEdges    = rank == 0 ? edges[2] : 0;
890     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0] + 1) : 0;
891     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1] + 1) : 0;
892     const PetscInt numZVertices = rank == 0 ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2] + 1) : 0;
893     const PetscInt numCells     = numXEdges * numYEdges * numZEdges;
894     const PetscInt numXFaces    = numYEdges * numZEdges;
895     const PetscInt numYFaces    = numXEdges * numZEdges;
896     const PetscInt numZFaces    = numXEdges * numYEdges;
897     const PetscInt numTotXFaces = numXVertices * numXFaces;
898     const PetscInt numTotYFaces = numYVertices * numYFaces;
899     const PetscInt numTotZFaces = numZVertices * numZFaces;
900     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
901     const PetscInt numTotXEdges = numXEdges * numYVertices * numZVertices;
902     const PetscInt numTotYEdges = numYEdges * numXVertices * numZVertices;
903     const PetscInt numTotZEdges = numZEdges * numXVertices * numYVertices;
904     const PetscInt numVertices  = numXVertices * numYVertices * numZVertices;
905     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
906     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
907     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
908     const PetscInt firstYFace   = firstXFace + numTotXFaces;
909     const PetscInt firstZFace   = firstYFace + numTotYFaces;
910     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
911     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
912     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
913     Vec            coordinates;
914     PetscSection   coordSection;
915     PetscScalar   *coords;
916     PetscInt       coordSize;
917     PetscInt       v, vx, vy, vz;
918     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
919 
920     PetscCall(DMPlexSetChart(dm, 0, numCells + numFaces + numEdges + numVertices));
921     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
922     for (f = firstXFace; f < firstXFace + numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
923     for (e = firstXEdge; e < firstXEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
924     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
925     /* Build cells */
926     for (fz = 0; fz < numZEdges; ++fz) {
927       for (fy = 0; fy < numYEdges; ++fy) {
928         for (fx = 0; fx < numXEdges; ++fx) {
929           PetscInt cell  = (fz * numYEdges + fy) * numXEdges + fx;
930           PetscInt faceB = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
931           PetscInt faceT = firstZFace + (fy * numXEdges + fx) * numZVertices + ((fz + 1) % numZVertices);
932           PetscInt faceF = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
933           PetscInt faceK = firstYFace + (fz * numXEdges + fx) * numYVertices + ((fy + 1) % numYVertices);
934           PetscInt faceL = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
935           PetscInt faceR = firstXFace + (fz * numYEdges + fy) * numXVertices + ((fx + 1) % numXVertices);
936           /* B,  T,  F,  K,  R,  L */
937           PetscInt ornt[6] = {-2, 0, 0, -3, 0, -2}; /* ??? */
938           PetscInt cone[6];
939 
940           /* no boundary twisting in 3D */
941           cone[0] = faceB;
942           cone[1] = faceT;
943           cone[2] = faceF;
944           cone[3] = faceK;
945           cone[4] = faceR;
946           cone[5] = faceL;
947           PetscCall(DMPlexSetCone(dm, cell, cone));
948           PetscCall(DMPlexSetConeOrientation(dm, cell, ornt));
949           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
950           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
951           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
952         }
953       }
954     }
955     /* Build x faces */
956     for (fz = 0; fz < numZEdges; ++fz) {
957       for (fy = 0; fy < numYEdges; ++fy) {
958         for (fx = 0; fx < numXVertices; ++fx) {
959           PetscInt face    = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
960           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
961           PetscInt edgeR   = firstZEdge + (((fy + 1) % numYVertices) * numXVertices + fx) * numZEdges + fz;
962           PetscInt edgeB   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
963           PetscInt edgeT   = firstYEdge + (((fz + 1) % numZVertices) * numXVertices + fx) * numYEdges + fy;
964           PetscInt ornt[4] = {0, 0, -1, -1};
965           PetscInt cone[4];
966 
967           if (dim == 3) {
968             /* markers */
969             if (bdX != DM_BOUNDARY_PERIODIC) {
970               if (fx == numXVertices - 1) {
971                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight));
972                 PetscCall(DMSetLabelValue(dm, "marker", face, markerRight));
973               } else if (fx == 0) {
974                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft));
975                 PetscCall(DMSetLabelValue(dm, "marker", face, markerLeft));
976               }
977             }
978           }
979           cone[0] = edgeB;
980           cone[1] = edgeR;
981           cone[2] = edgeT;
982           cone[3] = edgeL;
983           PetscCall(DMPlexSetCone(dm, face, cone));
984           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
985         }
986       }
987     }
988     /* Build y faces */
989     for (fz = 0; fz < numZEdges; ++fz) {
990       for (fx = 0; fx < numXEdges; ++fx) {
991         for (fy = 0; fy < numYVertices; ++fy) {
992           PetscInt face    = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
993           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
994           PetscInt edgeR   = firstZEdge + (fy * numXVertices + ((fx + 1) % numXVertices)) * numZEdges + fz;
995           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
996           PetscInt edgeT   = firstXEdge + (((fz + 1) % numZVertices) * numYVertices + fy) * numXEdges + fx;
997           PetscInt ornt[4] = {0, 0, -1, -1};
998           PetscInt cone[4];
999 
1000           if (dim == 3) {
1001             /* markers */
1002             if (bdY != DM_BOUNDARY_PERIODIC) {
1003               if (fy == numYVertices - 1) {
1004                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack));
1005                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBack));
1006               } else if (fy == 0) {
1007                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront));
1008                 PetscCall(DMSetLabelValue(dm, "marker", face, markerFront));
1009               }
1010             }
1011           }
1012           cone[0] = edgeB;
1013           cone[1] = edgeR;
1014           cone[2] = edgeT;
1015           cone[3] = edgeL;
1016           PetscCall(DMPlexSetCone(dm, face, cone));
1017           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1018         }
1019       }
1020     }
1021     /* Build z faces */
1022     for (fy = 0; fy < numYEdges; ++fy) {
1023       for (fx = 0; fx < numXEdges; ++fx) {
1024         for (fz = 0; fz < numZVertices; fz++) {
1025           PetscInt face    = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
1026           PetscInt edgeL   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
1027           PetscInt edgeR   = firstYEdge + (fz * numXVertices + ((fx + 1) % numXVertices)) * numYEdges + fy;
1028           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1029           PetscInt edgeT   = firstXEdge + (fz * numYVertices + ((fy + 1) % numYVertices)) * numXEdges + fx;
1030           PetscInt ornt[4] = {0, 0, -1, -1};
1031           PetscInt cone[4];
1032 
1033           if (dim == 2) {
1034             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges - 1) {
1035               edgeR += numYEdges - 1 - 2 * fy;
1036               ornt[1] = -1;
1037             }
1038             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges - 1) {
1039               edgeT += numXEdges - 1 - 2 * fx;
1040               ornt[2] = 0;
1041             }
1042             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1043             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1044           } else {
1045             /* markers */
1046             if (bdZ != DM_BOUNDARY_PERIODIC) {
1047               if (fz == numZVertices - 1) {
1048                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop));
1049                 PetscCall(DMSetLabelValue(dm, "marker", face, markerTop));
1050               } else if (fz == 0) {
1051                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom));
1052                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBottom));
1053               }
1054             }
1055           }
1056           cone[0] = edgeB;
1057           cone[1] = edgeR;
1058           cone[2] = edgeT;
1059           cone[3] = edgeL;
1060           PetscCall(DMPlexSetCone(dm, face, cone));
1061           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1062         }
1063       }
1064     }
1065     /* Build Z edges*/
1066     for (vy = 0; vy < numYVertices; vy++) {
1067       for (vx = 0; vx < numXVertices; vx++) {
1068         for (ez = 0; ez < numZEdges; ez++) {
1069           const PetscInt edge    = firstZEdge + (vy * numXVertices + vx) * numZEdges + ez;
1070           const PetscInt vertexB = firstVertex + (ez * numYVertices + vy) * numXVertices + vx;
1071           const PetscInt vertexT = firstVertex + (((ez + 1) % numZVertices) * numYVertices + vy) * numXVertices + vx;
1072           PetscInt       cone[2];
1073 
1074           cone[0] = vertexB;
1075           cone[1] = vertexT;
1076           PetscCall(DMPlexSetCone(dm, edge, cone));
1077           if (dim == 3) {
1078             if (bdX != DM_BOUNDARY_PERIODIC) {
1079               if (vx == numXVertices - 1) {
1080                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1081                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1082                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1083               } else if (vx == 0) {
1084                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1085                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1086                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1087               }
1088             }
1089             if (bdY != DM_BOUNDARY_PERIODIC) {
1090               if (vy == numYVertices - 1) {
1091                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1092                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1093                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1094               } else if (vy == 0) {
1095                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1096                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1097                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1098               }
1099             }
1100           }
1101         }
1102       }
1103     }
1104     /* Build Y edges*/
1105     for (vz = 0; vz < numZVertices; vz++) {
1106       for (vx = 0; vx < numXVertices; vx++) {
1107         for (ey = 0; ey < numYEdges; ey++) {
1108           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges - 1) ? (numXVertices - vx - 1) : (vz * numYVertices + ((ey + 1) % numYVertices)) * numXVertices + vx;
1109           const PetscInt edge    = firstYEdge + (vz * numXVertices + vx) * numYEdges + ey;
1110           const PetscInt vertexF = firstVertex + (vz * numYVertices + ey) * numXVertices + vx;
1111           const PetscInt vertexK = firstVertex + nextv;
1112           PetscInt       cone[2];
1113 
1114           cone[0] = vertexF;
1115           cone[1] = vertexK;
1116           PetscCall(DMPlexSetCone(dm, edge, cone));
1117           if (dim == 2) {
1118             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1119               if (vx == numXVertices - 1) {
1120                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight));
1121                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1122                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1123                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1124               } else if (vx == 0) {
1125                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft));
1126                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1127                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1128                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1129               }
1130             } else {
1131               if (vx == 0 && cutLabel) {
1132                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1133                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1134                 if (ey == numYEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1135               }
1136             }
1137           } else {
1138             if (bdX != DM_BOUNDARY_PERIODIC) {
1139               if (vx == numXVertices - 1) {
1140                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1141                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1142                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1143               } else if (vx == 0) {
1144                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1145                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1146                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1147               }
1148             }
1149             if (bdZ != DM_BOUNDARY_PERIODIC) {
1150               if (vz == numZVertices - 1) {
1151                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1152                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1153                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1154               } else if (vz == 0) {
1155                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1156                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1157                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1158               }
1159             }
1160           }
1161         }
1162       }
1163     }
1164     /* Build X edges*/
1165     for (vz = 0; vz < numZVertices; vz++) {
1166       for (vy = 0; vy < numYVertices; vy++) {
1167         for (ex = 0; ex < numXEdges; ex++) {
1168           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges - 1) ? (numYVertices - vy - 1) * numXVertices : (vz * numYVertices + vy) * numXVertices + (ex + 1) % numXVertices;
1169           const PetscInt edge    = firstXEdge + (vz * numYVertices + vy) * numXEdges + ex;
1170           const PetscInt vertexL = firstVertex + (vz * numYVertices + vy) * numXVertices + ex;
1171           const PetscInt vertexR = firstVertex + nextv;
1172           PetscInt       cone[2];
1173 
1174           cone[0] = vertexL;
1175           cone[1] = vertexR;
1176           PetscCall(DMPlexSetCone(dm, edge, cone));
1177           if (dim == 2) {
1178             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1179               if (vy == numYVertices - 1) {
1180                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop));
1181                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1182                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1183                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1184               } else if (vy == 0) {
1185                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom));
1186                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1187                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1188                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1189               }
1190             } else {
1191               if (vy == 0 && cutLabel) {
1192                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1193                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1194                 if (ex == numXEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1195               }
1196             }
1197           } else {
1198             if (bdY != DM_BOUNDARY_PERIODIC) {
1199               if (vy == numYVertices - 1) {
1200                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1201                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1202                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1203               } else if (vy == 0) {
1204                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1205                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1206                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1207               }
1208             }
1209             if (bdZ != DM_BOUNDARY_PERIODIC) {
1210               if (vz == numZVertices - 1) {
1211                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1212                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1213                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1214               } else if (vz == 0) {
1215                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1216                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1217                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1218               }
1219             }
1220           }
1221         }
1222       }
1223     }
1224     PetscCall(DMPlexSymmetrize(dm));
1225     PetscCall(DMPlexStratify(dm));
1226     /* Build coordinates */
1227     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1228     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1229     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1230     PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVertices));
1231     for (v = firstVertex; v < firstVertex + numVertices; ++v) {
1232       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1233       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1234     }
1235     PetscCall(PetscSectionSetUp(coordSection));
1236     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1237     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1238     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1239     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1240     PetscCall(VecSetBlockSize(coordinates, dim));
1241     PetscCall(VecSetType(coordinates, VECSTANDARD));
1242     PetscCall(VecGetArray(coordinates, &coords));
1243     for (vz = 0; vz < numZVertices; ++vz) {
1244       for (vy = 0; vy < numYVertices; ++vy) {
1245         for (vx = 0; vx < numXVertices; ++vx) {
1246           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * vx;
1247           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * vy;
1248           if (dim == 3) coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 2] = lower[2] + ((upper[2] - lower[2]) / numZEdges) * vz;
1249         }
1250       }
1251     }
1252     PetscCall(VecRestoreArray(coordinates, &coords));
1253     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1254     PetscCall(VecDestroy(&coordinates));
1255   }
1256   PetscFunctionReturn(0);
1257 }
1258 
1259 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1260 {
1261   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1262   PetscInt       fac[3] = {0, 0, 0}, d;
1263 
1264   PetscFunctionBegin;
1265   PetscValidPointer(dm, 1);
1266   PetscValidLogicalCollectiveInt(dm, dim, 2);
1267   PetscCall(DMSetDimension(dm, dim));
1268   for (d = 0; d < dim; ++d) {
1269     fac[d] = faces[d];
1270     bdt[d] = periodicity[d];
1271   }
1272   PetscCall(DMPlexCreateCubeMesh_Internal(dm, lower, upper, fac, bdt[0], bdt[1], bdt[2]));
1273   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST || periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST || (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
1274     PetscReal L[3]       = {-1., -1., 0.};
1275     PetscReal maxCell[3] = {-1., -1., 0.};
1276 
1277     for (d = 0; d < dim; ++d) {
1278       if (periodicity[d] != DM_BOUNDARY_NONE) {
1279         L[d]       = upper[d] - lower[d];
1280         maxCell[d] = 1.1 * (L[d] / PetscMax(1, faces[d]));
1281       }
1282     }
1283     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1284   }
1285   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
1286   PetscFunctionReturn(0);
1287 }
1288 
1289 static PetscErrorCode DMPlexCreateBoxMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1290 {
1291   PetscFunctionBegin;
1292   if (dim == 1) PetscCall(DMPlexCreateLineMesh_Internal(dm, faces[0], lower[0], upper[0], periodicity[0]));
1293   else if (simplex) PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1294   else PetscCall(DMPlexCreateBoxMesh_Tensor_Internal(dm, dim, faces, lower, upper, periodicity));
1295   if (!interpolate && dim > 1 && !simplex) {
1296     DM udm;
1297 
1298     PetscCall(DMPlexUninterpolate(dm, &udm));
1299     PetscCall(DMPlexCopyCoordinates(dm, udm));
1300     PetscCall(DMPlexReplace_Internal(dm, &udm));
1301   }
1302   PetscFunctionReturn(0);
1303 }
1304 
1305 /*@C
1306   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
1307 
1308   Collective
1309 
1310   Input Parameters:
1311 + comm        - The communicator for the DM object
1312 . dim         - The spatial dimension
1313 . simplex     - PETSC_TRUE for simplices, PETSC_FALSE for tensor cells
1314 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1315 . lower       - The lower left corner, or NULL for (0, 0, 0)
1316 . upper       - The upper right corner, or NULL for (1, 1, 1)
1317 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
1318 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1319 
1320   Output Parameter:
1321 . dm  - The DM object
1322 
1323   Note: If you want to customize this mesh using options, you just need to
1324 $  DMCreate(comm, &dm);
1325 $  DMSetType(dm, DMPLEX);
1326 $  DMSetFromOptions(dm);
1327 and use the options on the DMSetFromOptions() page.
1328 
1329   Here is the numbering returned for 2 faces in each direction for tensor cells:
1330 $ 10---17---11---18----12
1331 $  |         |         |
1332 $  |         |         |
1333 $ 20    2   22    3    24
1334 $  |         |         |
1335 $  |         |         |
1336 $  7---15----8---16----9
1337 $  |         |         |
1338 $  |         |         |
1339 $ 19    0   21    1   23
1340 $  |         |         |
1341 $  |         |         |
1342 $  4---13----5---14----6
1343 
1344 and for simplicial cells
1345 
1346 $ 14----8---15----9----16
1347 $  |\     5  |\      7 |
1348 $  | \       | \       |
1349 $ 13   2    14    3    15
1350 $  | 4   \   | 6   \   |
1351 $  |       \ |       \ |
1352 $ 11----6---12----7----13
1353 $  |\        |\        |
1354 $  | \    1  | \     3 |
1355 $ 10   0    11    1    12
1356 $  | 0   \   | 2   \   |
1357 $  |       \ |       \ |
1358 $  8----4----9----5----10
1359 
1360   Level: beginner
1361 
1362 .seealso: `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
1363 @*/
1364 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
1365 {
1366   PetscInt       fac[3] = {1, 1, 1};
1367   PetscReal      low[3] = {0, 0, 0};
1368   PetscReal      upp[3] = {1, 1, 1};
1369   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1370 
1371   PetscFunctionBegin;
1372   PetscCall(DMCreate(comm, dm));
1373   PetscCall(DMSetType(*dm, DMPLEX));
1374   PetscCall(DMPlexCreateBoxMesh_Internal(*dm, dim, simplex, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt, interpolate));
1375   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1376   PetscFunctionReturn(0);
1377 }
1378 
1379 static PetscErrorCode DMPlexCreateWedgeBoxMesh_Internal(DM dm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1380 {
1381   DM       bdm, vol;
1382   PetscInt i;
1383 
1384   PetscFunctionBegin;
1385   for (i = 0; i < 3; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity not yet supported");
1386   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &bdm));
1387   PetscCall(DMSetType(bdm, DMPLEX));
1388   PetscCall(DMSetDimension(bdm, 2));
1389   PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(bdm, 2, faces, lower, upper, periodicity, PETSC_TRUE));
1390   PetscCall(DMPlexExtrude(bdm, faces[2], upper[2] - lower[2], PETSC_TRUE, PETSC_FALSE, NULL, NULL, &vol));
1391   PetscCall(DMDestroy(&bdm));
1392   PetscCall(DMPlexReplace_Internal(dm, &vol));
1393   if (lower[2] != 0.0) {
1394     Vec          v;
1395     PetscScalar *x;
1396     PetscInt     cDim, n;
1397 
1398     PetscCall(DMGetCoordinatesLocal(dm, &v));
1399     PetscCall(VecGetBlockSize(v, &cDim));
1400     PetscCall(VecGetLocalSize(v, &n));
1401     PetscCall(VecGetArray(v, &x));
1402     x += cDim;
1403     for (i = 0; i < n; i += cDim) x[i] += lower[2];
1404     PetscCall(VecRestoreArray(v, &x));
1405     PetscCall(DMSetCoordinatesLocal(dm, v));
1406   }
1407   PetscFunctionReturn(0);
1408 }
1409 
1410 /*@
1411   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tesselating the (x,y) plane and extruding in the third direction using wedge cells.
1412 
1413   Collective
1414 
1415   Input Parameters:
1416 + comm        - The communicator for the DM object
1417 . faces       - Number of faces per dimension, or NULL for (1, 1, 1)
1418 . lower       - The lower left corner, or NULL for (0, 0, 0)
1419 . upper       - The upper right corner, or NULL for (1, 1, 1)
1420 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
1421 . orderHeight - If PETSC_TRUE, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1422 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1423 
1424   Output Parameter:
1425 . dm  - The DM object
1426 
1427   Level: beginner
1428 
1429 .seealso: `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateWedgeCylinderMesh()`, `DMExtrude()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1430 @*/
1431 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool orderHeight, PetscBool interpolate, DM *dm)
1432 {
1433   PetscInt       fac[3] = {1, 1, 1};
1434   PetscReal      low[3] = {0, 0, 0};
1435   PetscReal      upp[3] = {1, 1, 1};
1436   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1437 
1438   PetscFunctionBegin;
1439   PetscCall(DMCreate(comm, dm));
1440   PetscCall(DMSetType(*dm, DMPLEX));
1441   PetscCall(DMPlexCreateWedgeBoxMesh_Internal(*dm, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt));
1442   if (!interpolate) {
1443     DM udm;
1444 
1445     PetscCall(DMPlexUninterpolate(*dm, &udm));
1446     PetscCall(DMPlexReplace_Internal(*dm, &udm));
1447   }
1448   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1449   PetscFunctionReturn(0);
1450 }
1451 
1452 /*@C
1453   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all DM options in the database.
1454 
1455   Logically Collective on dm
1456 
1457   Input Parameters:
1458 + dm - the DM context
1459 - prefix - the prefix to prepend to all option names
1460 
1461   Notes:
1462   A hyphen (-) must NOT be given at the beginning of the prefix name.
1463   The first character of all runtime options is AUTOMATICALLY the hyphen.
1464 
1465   Level: advanced
1466 
1467 .seealso: `SNESSetFromOptions()`
1468 @*/
1469 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
1470 {
1471   DM_Plex *mesh = (DM_Plex *)dm->data;
1472 
1473   PetscFunctionBegin;
1474   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1475   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, prefix));
1476   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)mesh->partitioner, prefix));
1477   PetscFunctionReturn(0);
1478 }
1479 
1480 /* Remap geometry to cylinder
1481    TODO: This only works for a single refinement, then it is broken
1482 
1483      Interior square: Linear interpolation is correct
1484      The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
1485      such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
1486 
1487        phi     = arctan(y/x)
1488        d_close = sqrt(1/8 + 1/4 sin^2(phi))
1489        d_far   = sqrt(1/2 + sin^2(phi))
1490 
1491      so we remap them using
1492 
1493        x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
1494        y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
1495 
1496      If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
1497 */
1498 static void snapToCylinder(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
1499 {
1500   const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1501   const PetscReal ds2 = 0.5 * dis;
1502 
1503   if ((PetscAbsScalar(u[0]) <= ds2) && (PetscAbsScalar(u[1]) <= ds2)) {
1504     f0[0] = u[0];
1505     f0[1] = u[1];
1506   } else {
1507     PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
1508 
1509     x    = PetscRealPart(u[0]);
1510     y    = PetscRealPart(u[1]);
1511     phi  = PetscAtan2Real(y, x);
1512     sinp = PetscSinReal(phi);
1513     cosp = PetscCosReal(phi);
1514     if ((PetscAbsReal(phi) > PETSC_PI / 4.0) && (PetscAbsReal(phi) < 3.0 * PETSC_PI / 4.0)) {
1515       dc = PetscAbsReal(ds2 / sinp);
1516       df = PetscAbsReal(dis / sinp);
1517       xc = ds2 * x / PetscAbsReal(y);
1518       yc = ds2 * PetscSignReal(y);
1519     } else {
1520       dc = PetscAbsReal(ds2 / cosp);
1521       df = PetscAbsReal(dis / cosp);
1522       xc = ds2 * PetscSignReal(x);
1523       yc = ds2 * y / PetscAbsReal(x);
1524     }
1525     f0[0] = xc + (u[0] - xc) * (1.0 - dc) / (df - dc);
1526     f0[1] = yc + (u[1] - yc) * (1.0 - dc) / (df - dc);
1527   }
1528   f0[2] = u[2];
1529 }
1530 
1531 static PetscErrorCode DMPlexCreateHexCylinderMesh_Internal(DM dm, DMBoundaryType periodicZ)
1532 {
1533   const PetscInt dim = 3;
1534   PetscInt       numCells, numVertices;
1535   PetscMPIInt    rank;
1536 
1537   PetscFunctionBegin;
1538   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1539   PetscCall(DMSetDimension(dm, dim));
1540   /* Create topology */
1541   {
1542     PetscInt cone[8], c;
1543 
1544     numCells    = rank == 0 ? 5 : 0;
1545     numVertices = rank == 0 ? 16 : 0;
1546     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1547       numCells *= 3;
1548       numVertices = rank == 0 ? 24 : 0;
1549     }
1550     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1551     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 8));
1552     PetscCall(DMSetUp(dm));
1553     if (rank == 0) {
1554       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1555         cone[0] = 15;
1556         cone[1] = 18;
1557         cone[2] = 17;
1558         cone[3] = 16;
1559         cone[4] = 31;
1560         cone[5] = 32;
1561         cone[6] = 33;
1562         cone[7] = 34;
1563         PetscCall(DMPlexSetCone(dm, 0, cone));
1564         cone[0] = 16;
1565         cone[1] = 17;
1566         cone[2] = 24;
1567         cone[3] = 23;
1568         cone[4] = 32;
1569         cone[5] = 36;
1570         cone[6] = 37;
1571         cone[7] = 33; /* 22 25 26 21 */
1572         PetscCall(DMPlexSetCone(dm, 1, cone));
1573         cone[0] = 18;
1574         cone[1] = 27;
1575         cone[2] = 24;
1576         cone[3] = 17;
1577         cone[4] = 34;
1578         cone[5] = 33;
1579         cone[6] = 37;
1580         cone[7] = 38;
1581         PetscCall(DMPlexSetCone(dm, 2, cone));
1582         cone[0] = 29;
1583         cone[1] = 27;
1584         cone[2] = 18;
1585         cone[3] = 15;
1586         cone[4] = 35;
1587         cone[5] = 31;
1588         cone[6] = 34;
1589         cone[7] = 38;
1590         PetscCall(DMPlexSetCone(dm, 3, cone));
1591         cone[0] = 29;
1592         cone[1] = 15;
1593         cone[2] = 16;
1594         cone[3] = 23;
1595         cone[4] = 35;
1596         cone[5] = 36;
1597         cone[6] = 32;
1598         cone[7] = 31;
1599         PetscCall(DMPlexSetCone(dm, 4, cone));
1600 
1601         cone[0] = 31;
1602         cone[1] = 34;
1603         cone[2] = 33;
1604         cone[3] = 32;
1605         cone[4] = 19;
1606         cone[5] = 22;
1607         cone[6] = 21;
1608         cone[7] = 20;
1609         PetscCall(DMPlexSetCone(dm, 5, cone));
1610         cone[0] = 32;
1611         cone[1] = 33;
1612         cone[2] = 37;
1613         cone[3] = 36;
1614         cone[4] = 22;
1615         cone[5] = 25;
1616         cone[6] = 26;
1617         cone[7] = 21;
1618         PetscCall(DMPlexSetCone(dm, 6, cone));
1619         cone[0] = 34;
1620         cone[1] = 38;
1621         cone[2] = 37;
1622         cone[3] = 33;
1623         cone[4] = 20;
1624         cone[5] = 21;
1625         cone[6] = 26;
1626         cone[7] = 28;
1627         PetscCall(DMPlexSetCone(dm, 7, cone));
1628         cone[0] = 35;
1629         cone[1] = 38;
1630         cone[2] = 34;
1631         cone[3] = 31;
1632         cone[4] = 30;
1633         cone[5] = 19;
1634         cone[6] = 20;
1635         cone[7] = 28;
1636         PetscCall(DMPlexSetCone(dm, 8, cone));
1637         cone[0] = 35;
1638         cone[1] = 31;
1639         cone[2] = 32;
1640         cone[3] = 36;
1641         cone[4] = 30;
1642         cone[5] = 25;
1643         cone[6] = 22;
1644         cone[7] = 19;
1645         PetscCall(DMPlexSetCone(dm, 9, cone));
1646 
1647         cone[0] = 19;
1648         cone[1] = 20;
1649         cone[2] = 21;
1650         cone[3] = 22;
1651         cone[4] = 15;
1652         cone[5] = 16;
1653         cone[6] = 17;
1654         cone[7] = 18;
1655         PetscCall(DMPlexSetCone(dm, 10, cone));
1656         cone[0] = 22;
1657         cone[1] = 21;
1658         cone[2] = 26;
1659         cone[3] = 25;
1660         cone[4] = 16;
1661         cone[5] = 23;
1662         cone[6] = 24;
1663         cone[7] = 17;
1664         PetscCall(DMPlexSetCone(dm, 11, cone));
1665         cone[0] = 20;
1666         cone[1] = 28;
1667         cone[2] = 26;
1668         cone[3] = 21;
1669         cone[4] = 18;
1670         cone[5] = 17;
1671         cone[6] = 24;
1672         cone[7] = 27;
1673         PetscCall(DMPlexSetCone(dm, 12, cone));
1674         cone[0] = 30;
1675         cone[1] = 28;
1676         cone[2] = 20;
1677         cone[3] = 19;
1678         cone[4] = 29;
1679         cone[5] = 15;
1680         cone[6] = 18;
1681         cone[7] = 27;
1682         PetscCall(DMPlexSetCone(dm, 13, cone));
1683         cone[0] = 30;
1684         cone[1] = 19;
1685         cone[2] = 22;
1686         cone[3] = 25;
1687         cone[4] = 29;
1688         cone[5] = 23;
1689         cone[6] = 16;
1690         cone[7] = 15;
1691         PetscCall(DMPlexSetCone(dm, 14, cone));
1692       } else {
1693         cone[0] = 5;
1694         cone[1] = 8;
1695         cone[2] = 7;
1696         cone[3] = 6;
1697         cone[4] = 9;
1698         cone[5] = 12;
1699         cone[6] = 11;
1700         cone[7] = 10;
1701         PetscCall(DMPlexSetCone(dm, 0, cone));
1702         cone[0] = 6;
1703         cone[1] = 7;
1704         cone[2] = 14;
1705         cone[3] = 13;
1706         cone[4] = 12;
1707         cone[5] = 15;
1708         cone[6] = 16;
1709         cone[7] = 11;
1710         PetscCall(DMPlexSetCone(dm, 1, cone));
1711         cone[0] = 8;
1712         cone[1] = 17;
1713         cone[2] = 14;
1714         cone[3] = 7;
1715         cone[4] = 10;
1716         cone[5] = 11;
1717         cone[6] = 16;
1718         cone[7] = 18;
1719         PetscCall(DMPlexSetCone(dm, 2, cone));
1720         cone[0] = 19;
1721         cone[1] = 17;
1722         cone[2] = 8;
1723         cone[3] = 5;
1724         cone[4] = 20;
1725         cone[5] = 9;
1726         cone[6] = 10;
1727         cone[7] = 18;
1728         PetscCall(DMPlexSetCone(dm, 3, cone));
1729         cone[0] = 19;
1730         cone[1] = 5;
1731         cone[2] = 6;
1732         cone[3] = 13;
1733         cone[4] = 20;
1734         cone[5] = 15;
1735         cone[6] = 12;
1736         cone[7] = 9;
1737         PetscCall(DMPlexSetCone(dm, 4, cone));
1738       }
1739     }
1740     PetscCall(DMPlexSymmetrize(dm));
1741     PetscCall(DMPlexStratify(dm));
1742   }
1743   /* Create cube geometry */
1744   {
1745     Vec             coordinates;
1746     PetscSection    coordSection;
1747     PetscScalar    *coords;
1748     PetscInt        coordSize, v;
1749     const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1750     const PetscReal ds2 = dis / 2.0;
1751 
1752     /* Build coordinates */
1753     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1754     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1755     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1756     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1757     for (v = numCells; v < numCells + numVertices; ++v) {
1758       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1759       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1760     }
1761     PetscCall(PetscSectionSetUp(coordSection));
1762     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1763     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1764     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1765     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1766     PetscCall(VecSetBlockSize(coordinates, dim));
1767     PetscCall(VecSetType(coordinates, VECSTANDARD));
1768     PetscCall(VecGetArray(coordinates, &coords));
1769     if (rank == 0) {
1770       coords[0 * dim + 0]  = -ds2;
1771       coords[0 * dim + 1]  = -ds2;
1772       coords[0 * dim + 2]  = 0.0;
1773       coords[1 * dim + 0]  = ds2;
1774       coords[1 * dim + 1]  = -ds2;
1775       coords[1 * dim + 2]  = 0.0;
1776       coords[2 * dim + 0]  = ds2;
1777       coords[2 * dim + 1]  = ds2;
1778       coords[2 * dim + 2]  = 0.0;
1779       coords[3 * dim + 0]  = -ds2;
1780       coords[3 * dim + 1]  = ds2;
1781       coords[3 * dim + 2]  = 0.0;
1782       coords[4 * dim + 0]  = -ds2;
1783       coords[4 * dim + 1]  = -ds2;
1784       coords[4 * dim + 2]  = 1.0;
1785       coords[5 * dim + 0]  = -ds2;
1786       coords[5 * dim + 1]  = ds2;
1787       coords[5 * dim + 2]  = 1.0;
1788       coords[6 * dim + 0]  = ds2;
1789       coords[6 * dim + 1]  = ds2;
1790       coords[6 * dim + 2]  = 1.0;
1791       coords[7 * dim + 0]  = ds2;
1792       coords[7 * dim + 1]  = -ds2;
1793       coords[7 * dim + 2]  = 1.0;
1794       coords[8 * dim + 0]  = dis;
1795       coords[8 * dim + 1]  = -dis;
1796       coords[8 * dim + 2]  = 0.0;
1797       coords[9 * dim + 0]  = dis;
1798       coords[9 * dim + 1]  = dis;
1799       coords[9 * dim + 2]  = 0.0;
1800       coords[10 * dim + 0] = dis;
1801       coords[10 * dim + 1] = -dis;
1802       coords[10 * dim + 2] = 1.0;
1803       coords[11 * dim + 0] = dis;
1804       coords[11 * dim + 1] = dis;
1805       coords[11 * dim + 2] = 1.0;
1806       coords[12 * dim + 0] = -dis;
1807       coords[12 * dim + 1] = dis;
1808       coords[12 * dim + 2] = 0.0;
1809       coords[13 * dim + 0] = -dis;
1810       coords[13 * dim + 1] = dis;
1811       coords[13 * dim + 2] = 1.0;
1812       coords[14 * dim + 0] = -dis;
1813       coords[14 * dim + 1] = -dis;
1814       coords[14 * dim + 2] = 0.0;
1815       coords[15 * dim + 0] = -dis;
1816       coords[15 * dim + 1] = -dis;
1817       coords[15 * dim + 2] = 1.0;
1818       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1819         /* 15 31 19 */ coords[16 * dim + 0] = -ds2;
1820         coords[16 * dim + 1]                = -ds2;
1821         coords[16 * dim + 2]                = 0.5;
1822         /* 16 32 22 */ coords[17 * dim + 0] = ds2;
1823         coords[17 * dim + 1]                = -ds2;
1824         coords[17 * dim + 2]                = 0.5;
1825         /* 17 33 21 */ coords[18 * dim + 0] = ds2;
1826         coords[18 * dim + 1]                = ds2;
1827         coords[18 * dim + 2]                = 0.5;
1828         /* 18 34 20 */ coords[19 * dim + 0] = -ds2;
1829         coords[19 * dim + 1]                = ds2;
1830         coords[19 * dim + 2]                = 0.5;
1831         /* 29 35 30 */ coords[20 * dim + 0] = -dis;
1832         coords[20 * dim + 1]                = -dis;
1833         coords[20 * dim + 2]                = 0.5;
1834         /* 23 36 25 */ coords[21 * dim + 0] = dis;
1835         coords[21 * dim + 1]                = -dis;
1836         coords[21 * dim + 2]                = 0.5;
1837         /* 24 37 26 */ coords[22 * dim + 0] = dis;
1838         coords[22 * dim + 1]                = dis;
1839         coords[22 * dim + 2]                = 0.5;
1840         /* 27 38 28 */ coords[23 * dim + 0] = -dis;
1841         coords[23 * dim + 1]                = dis;
1842         coords[23 * dim + 2]                = 0.5;
1843       }
1844     }
1845     PetscCall(VecRestoreArray(coordinates, &coords));
1846     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1847     PetscCall(VecDestroy(&coordinates));
1848   }
1849   /* Create periodicity */
1850   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1851     PetscReal L[3]       = {-1., -1., 0.};
1852     PetscReal maxCell[3] = {-1., -1., 0.};
1853     PetscReal lower[3]   = {0.0, 0.0, 0.0};
1854     PetscReal upper[3]   = {1.0, 1.0, 1.5};
1855     PetscInt  numZCells  = 3;
1856 
1857     L[2]       = upper[2] - lower[2];
1858     maxCell[2] = 1.1 * (L[2] / numZCells);
1859     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1860   }
1861   {
1862     DM          cdm;
1863     PetscDS     cds;
1864     PetscScalar c[2] = {1.0, 1.0};
1865 
1866     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToCylinder));
1867     PetscCall(DMGetCoordinateDM(dm, &cdm));
1868     PetscCall(DMGetDS(cdm, &cds));
1869     PetscCall(PetscDSSetConstants(cds, 2, c));
1870   }
1871   /* Wait for coordinate creation before doing in-place modification */
1872   PetscCall(DMPlexInterpolateInPlace_Internal(dm));
1873   PetscFunctionReturn(0);
1874 }
1875 
1876 /*@
1877   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
1878 
1879   Collective
1880 
1881   Input Parameters:
1882 + comm      - The communicator for the DM object
1883 - periodicZ - The boundary type for the Z direction
1884 
1885   Output Parameter:
1886 . dm  - The DM object
1887 
1888   Note:
1889   Here is the output numbering looking from the bottom of the cylinder:
1890 $       17-----14
1891 $        |     |
1892 $        |  2  |
1893 $        |     |
1894 $ 17-----8-----7-----14
1895 $  |     |     |     |
1896 $  |  3  |  0  |  1  |
1897 $  |     |     |     |
1898 $ 19-----5-----6-----13
1899 $        |     |
1900 $        |  4  |
1901 $        |     |
1902 $       19-----13
1903 $
1904 $ and up through the top
1905 $
1906 $       18-----16
1907 $        |     |
1908 $        |  2  |
1909 $        |     |
1910 $ 18----10----11-----16
1911 $  |     |     |     |
1912 $  |  3  |  0  |  1  |
1913 $  |     |     |     |
1914 $ 20-----9----12-----15
1915 $        |     |
1916 $        |  4  |
1917 $        |     |
1918 $       20-----15
1919 
1920   Level: beginner
1921 
1922 .seealso: `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1923 @*/
1924 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, DMBoundaryType periodicZ, DM *dm)
1925 {
1926   PetscFunctionBegin;
1927   PetscValidPointer(dm, 3);
1928   PetscCall(DMCreate(comm, dm));
1929   PetscCall(DMSetType(*dm, DMPLEX));
1930   PetscCall(DMPlexCreateHexCylinderMesh_Internal(*dm, periodicZ));
1931   PetscFunctionReturn(0);
1932 }
1933 
1934 static PetscErrorCode DMPlexCreateWedgeCylinderMesh_Internal(DM dm, PetscInt n, PetscBool interpolate)
1935 {
1936   const PetscInt dim = 3;
1937   PetscInt       numCells, numVertices, v;
1938   PetscMPIInt    rank;
1939 
1940   PetscFunctionBegin;
1941   PetscCheck(n >= 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %" PetscInt_FMT " cannot be negative", n);
1942   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1943   PetscCall(DMSetDimension(dm, dim));
1944   /* Must create the celltype label here so that we do not automatically try to compute the types */
1945   PetscCall(DMCreateLabel(dm, "celltype"));
1946   /* Create topology */
1947   {
1948     PetscInt cone[6], c;
1949 
1950     numCells    = rank == 0 ? n : 0;
1951     numVertices = rank == 0 ? 2 * (n + 1) : 0;
1952     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1953     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
1954     PetscCall(DMSetUp(dm));
1955     for (c = 0; c < numCells; c++) {
1956       cone[0] = c + n * 1;
1957       cone[1] = (c + 1) % n + n * 1;
1958       cone[2] = 0 + 3 * n;
1959       cone[3] = c + n * 2;
1960       cone[4] = (c + 1) % n + n * 2;
1961       cone[5] = 1 + 3 * n;
1962       PetscCall(DMPlexSetCone(dm, c, cone));
1963       PetscCall(DMPlexSetCellType(dm, c, DM_POLYTOPE_TRI_PRISM_TENSOR));
1964     }
1965     PetscCall(DMPlexSymmetrize(dm));
1966     PetscCall(DMPlexStratify(dm));
1967   }
1968   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetCellType(dm, v, DM_POLYTOPE_POINT));
1969   /* Create cylinder geometry */
1970   {
1971     Vec          coordinates;
1972     PetscSection coordSection;
1973     PetscScalar *coords;
1974     PetscInt     coordSize, c;
1975 
1976     /* Build coordinates */
1977     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1978     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1979     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1980     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1981     for (v = numCells; v < numCells + numVertices; ++v) {
1982       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1983       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1984     }
1985     PetscCall(PetscSectionSetUp(coordSection));
1986     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1987     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1988     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1989     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1990     PetscCall(VecSetBlockSize(coordinates, dim));
1991     PetscCall(VecSetType(coordinates, VECSTANDARD));
1992     PetscCall(VecGetArray(coordinates, &coords));
1993     for (c = 0; c < numCells; c++) {
1994       coords[(c + 0 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
1995       coords[(c + 0 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
1996       coords[(c + 0 * n) * dim + 2] = 1.0;
1997       coords[(c + 1 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
1998       coords[(c + 1 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
1999       coords[(c + 1 * n) * dim + 2] = 0.0;
2000     }
2001     if (rank == 0) {
2002       coords[(2 * n + 0) * dim + 0] = 0.0;
2003       coords[(2 * n + 0) * dim + 1] = 0.0;
2004       coords[(2 * n + 0) * dim + 2] = 1.0;
2005       coords[(2 * n + 1) * dim + 0] = 0.0;
2006       coords[(2 * n + 1) * dim + 1] = 0.0;
2007       coords[(2 * n + 1) * dim + 2] = 0.0;
2008     }
2009     PetscCall(VecRestoreArray(coordinates, &coords));
2010     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2011     PetscCall(VecDestroy(&coordinates));
2012   }
2013   /* Interpolate */
2014   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2015   PetscFunctionReturn(0);
2016 }
2017 
2018 /*@
2019   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
2020 
2021   Collective
2022 
2023   Input Parameters:
2024 + comm - The communicator for the DM object
2025 . n    - The number of wedges around the origin
2026 - interpolate - Create edges and faces
2027 
2028   Output Parameter:
2029 . dm  - The DM object
2030 
2031   Level: beginner
2032 
2033 .seealso: `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2034 @*/
2035 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
2036 {
2037   PetscFunctionBegin;
2038   PetscValidPointer(dm, 4);
2039   PetscCall(DMCreate(comm, dm));
2040   PetscCall(DMSetType(*dm, DMPLEX));
2041   PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(*dm, n, interpolate));
2042   PetscFunctionReturn(0);
2043 }
2044 
2045 static inline PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2046 {
2047   PetscReal prod = 0.0;
2048   PetscInt  i;
2049   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
2050   return PetscSqrtReal(prod);
2051 }
2052 static inline PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2053 {
2054   PetscReal prod = 0.0;
2055   PetscInt  i;
2056   for (i = 0; i < dim; ++i) prod += x[i] * y[i];
2057   return prod;
2058 }
2059 
2060 /* The first constant is the sphere radius */
2061 static void snapToSphere(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
2062 {
2063   PetscReal r     = PetscRealPart(constants[0]);
2064   PetscReal norm2 = 0.0, fac;
2065   PetscInt  n     = uOff[1] - uOff[0], d;
2066 
2067   for (d = 0; d < n; ++d) norm2 += PetscSqr(PetscRealPart(u[d]));
2068   fac = r / PetscSqrtReal(norm2);
2069   for (d = 0; d < n; ++d) f0[d] = u[d] * fac;
2070 }
2071 
2072 static PetscErrorCode DMPlexCreateSphereMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, PetscReal R)
2073 {
2074   const PetscInt embedDim = dim + 1;
2075   PetscSection   coordSection;
2076   Vec            coordinates;
2077   PetscScalar   *coords;
2078   PetscReal     *coordsIn;
2079   PetscInt       numCells, numEdges, numVerts = 0, firstVertex = 0, v, firstEdge, coordSize, d, c, e;
2080   PetscMPIInt    rank;
2081 
2082   PetscFunctionBegin;
2083   PetscValidLogicalCollectiveBool(dm, simplex, 3);
2084   PetscCall(DMSetDimension(dm, dim));
2085   PetscCall(DMSetCoordinateDim(dm, dim + 1));
2086   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2087   switch (dim) {
2088   case 2:
2089     if (simplex) {
2090       const PetscReal radius    = PetscSqrtReal(1 + PETSC_PHI * PETSC_PHI) / (1.0 + PETSC_PHI);
2091       const PetscReal edgeLen   = 2.0 / (1.0 + PETSC_PHI) * (R / radius);
2092       const PetscInt  degree    = 5;
2093       PetscReal       vertex[3] = {0.0, 1.0 / (1.0 + PETSC_PHI), PETSC_PHI / (1.0 + PETSC_PHI)};
2094       PetscInt        s[3]      = {1, 1, 1};
2095       PetscInt        cone[3];
2096       PetscInt       *graph, p, i, j, k;
2097 
2098       vertex[0] *= R / radius;
2099       vertex[1] *= R / radius;
2100       vertex[2] *= R / radius;
2101       numCells    = rank == 0 ? 20 : 0;
2102       numVerts    = rank == 0 ? 12 : 0;
2103       firstVertex = numCells;
2104       /* Use icosahedron, which for a R-sphere has coordinates which are all cyclic permutations of
2105 
2106            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
2107 
2108          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2109          length is then given by 2/(1+\phi) = 2 * 0.38197 = 0.76393.
2110       */
2111       /* Construct vertices */
2112       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2113       if (rank == 0) {
2114         for (p = 0, i = 0; p < embedDim; ++p) {
2115           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2116             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2117               for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertex[(d + p) % embedDim];
2118               ++i;
2119             }
2120           }
2121         }
2122       }
2123       /* Construct graph */
2124       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2125       for (i = 0; i < numVerts; ++i) {
2126         for (j = 0, k = 0; j < numVerts; ++j) {
2127           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2128             graph[i * numVerts + j] = 1;
2129             ++k;
2130           }
2131         }
2132         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid icosahedron, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2133       }
2134       /* Build Topology */
2135       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2136       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2137       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2138       /* Cells */
2139       for (i = 0, c = 0; i < numVerts; ++i) {
2140         for (j = 0; j < i; ++j) {
2141           for (k = 0; k < j; ++k) {
2142             if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i]) {
2143               cone[0] = firstVertex + i;
2144               cone[1] = firstVertex + j;
2145               cone[2] = firstVertex + k;
2146               /* Check orientation */
2147               {
2148                 const PetscInt epsilon[3][3][3] = {
2149                   {{0, 0, 0},  {0, 0, 1},  {0, -1, 0}},
2150                   {{0, 0, -1}, {0, 0, 0},  {1, 0, 0} },
2151                   {{0, 1, 0},  {-1, 0, 0}, {0, 0, 0} }
2152                 };
2153                 PetscReal normal[3];
2154                 PetscInt  e, f;
2155 
2156                 for (d = 0; d < embedDim; ++d) {
2157                   normal[d] = 0.0;
2158                   for (e = 0; e < embedDim; ++e) {
2159                     for (f = 0; f < embedDim; ++f) normal[d] += epsilon[d][e][f] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]);
2160                   }
2161                 }
2162                 if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2163                   PetscInt tmp = cone[1];
2164                   cone[1]      = cone[2];
2165                   cone[2]      = tmp;
2166                 }
2167               }
2168               PetscCall(DMPlexSetCone(dm, c++, cone));
2169             }
2170           }
2171         }
2172       }
2173       PetscCall(DMPlexSymmetrize(dm));
2174       PetscCall(DMPlexStratify(dm));
2175       PetscCall(PetscFree(graph));
2176     } else {
2177       /*
2178         12-21--13
2179          |     |
2180         25  4  24
2181          |     |
2182   12-25--9-16--8-24--13
2183    |     |     |     |
2184   23  5 17  0 15  3  22
2185    |     |     |     |
2186   10-20--6-14--7-19--11
2187          |     |
2188         20  1  19
2189          |     |
2190         10-18--11
2191          |     |
2192         23  2  22
2193          |     |
2194         12-21--13
2195        */
2196       PetscInt cone[4], ornt[4];
2197 
2198       numCells    = rank == 0 ? 6 : 0;
2199       numEdges    = rank == 0 ? 12 : 0;
2200       numVerts    = rank == 0 ? 8 : 0;
2201       firstVertex = numCells;
2202       firstEdge   = numCells + numVerts;
2203       /* Build Topology */
2204       PetscCall(DMPlexSetChart(dm, 0, numCells + numEdges + numVerts));
2205       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 4));
2206       for (e = firstEdge; e < firstEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
2207       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2208       if (rank == 0) {
2209         /* Cell 0 */
2210         cone[0] = 14;
2211         cone[1] = 15;
2212         cone[2] = 16;
2213         cone[3] = 17;
2214         PetscCall(DMPlexSetCone(dm, 0, cone));
2215         ornt[0] = 0;
2216         ornt[1] = 0;
2217         ornt[2] = 0;
2218         ornt[3] = 0;
2219         PetscCall(DMPlexSetConeOrientation(dm, 0, ornt));
2220         /* Cell 1 */
2221         cone[0] = 18;
2222         cone[1] = 19;
2223         cone[2] = 14;
2224         cone[3] = 20;
2225         PetscCall(DMPlexSetCone(dm, 1, cone));
2226         ornt[0] = 0;
2227         ornt[1] = 0;
2228         ornt[2] = -1;
2229         ornt[3] = 0;
2230         PetscCall(DMPlexSetConeOrientation(dm, 1, ornt));
2231         /* Cell 2 */
2232         cone[0] = 21;
2233         cone[1] = 22;
2234         cone[2] = 18;
2235         cone[3] = 23;
2236         PetscCall(DMPlexSetCone(dm, 2, cone));
2237         ornt[0] = 0;
2238         ornt[1] = 0;
2239         ornt[2] = -1;
2240         ornt[3] = 0;
2241         PetscCall(DMPlexSetConeOrientation(dm, 2, ornt));
2242         /* Cell 3 */
2243         cone[0] = 19;
2244         cone[1] = 22;
2245         cone[2] = 24;
2246         cone[3] = 15;
2247         PetscCall(DMPlexSetCone(dm, 3, cone));
2248         ornt[0] = -1;
2249         ornt[1] = -1;
2250         ornt[2] = 0;
2251         ornt[3] = -1;
2252         PetscCall(DMPlexSetConeOrientation(dm, 3, ornt));
2253         /* Cell 4 */
2254         cone[0] = 16;
2255         cone[1] = 24;
2256         cone[2] = 21;
2257         cone[3] = 25;
2258         PetscCall(DMPlexSetCone(dm, 4, cone));
2259         ornt[0] = -1;
2260         ornt[1] = -1;
2261         ornt[2] = -1;
2262         ornt[3] = 0;
2263         PetscCall(DMPlexSetConeOrientation(dm, 4, ornt));
2264         /* Cell 5 */
2265         cone[0] = 20;
2266         cone[1] = 17;
2267         cone[2] = 25;
2268         cone[3] = 23;
2269         PetscCall(DMPlexSetCone(dm, 5, cone));
2270         ornt[0] = -1;
2271         ornt[1] = -1;
2272         ornt[2] = -1;
2273         ornt[3] = -1;
2274         PetscCall(DMPlexSetConeOrientation(dm, 5, ornt));
2275         /* Edges */
2276         cone[0] = 6;
2277         cone[1] = 7;
2278         PetscCall(DMPlexSetCone(dm, 14, cone));
2279         cone[0] = 7;
2280         cone[1] = 8;
2281         PetscCall(DMPlexSetCone(dm, 15, cone));
2282         cone[0] = 8;
2283         cone[1] = 9;
2284         PetscCall(DMPlexSetCone(dm, 16, cone));
2285         cone[0] = 9;
2286         cone[1] = 6;
2287         PetscCall(DMPlexSetCone(dm, 17, cone));
2288         cone[0] = 10;
2289         cone[1] = 11;
2290         PetscCall(DMPlexSetCone(dm, 18, cone));
2291         cone[0] = 11;
2292         cone[1] = 7;
2293         PetscCall(DMPlexSetCone(dm, 19, cone));
2294         cone[0] = 6;
2295         cone[1] = 10;
2296         PetscCall(DMPlexSetCone(dm, 20, cone));
2297         cone[0] = 12;
2298         cone[1] = 13;
2299         PetscCall(DMPlexSetCone(dm, 21, cone));
2300         cone[0] = 13;
2301         cone[1] = 11;
2302         PetscCall(DMPlexSetCone(dm, 22, cone));
2303         cone[0] = 10;
2304         cone[1] = 12;
2305         PetscCall(DMPlexSetCone(dm, 23, cone));
2306         cone[0] = 13;
2307         cone[1] = 8;
2308         PetscCall(DMPlexSetCone(dm, 24, cone));
2309         cone[0] = 12;
2310         cone[1] = 9;
2311         PetscCall(DMPlexSetCone(dm, 25, cone));
2312       }
2313       PetscCall(DMPlexSymmetrize(dm));
2314       PetscCall(DMPlexStratify(dm));
2315       /* Build coordinates */
2316       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2317       if (rank == 0) {
2318         coordsIn[0 * embedDim + 0] = -R;
2319         coordsIn[0 * embedDim + 1] = R;
2320         coordsIn[0 * embedDim + 2] = -R;
2321         coordsIn[1 * embedDim + 0] = R;
2322         coordsIn[1 * embedDim + 1] = R;
2323         coordsIn[1 * embedDim + 2] = -R;
2324         coordsIn[2 * embedDim + 0] = R;
2325         coordsIn[2 * embedDim + 1] = -R;
2326         coordsIn[2 * embedDim + 2] = -R;
2327         coordsIn[3 * embedDim + 0] = -R;
2328         coordsIn[3 * embedDim + 1] = -R;
2329         coordsIn[3 * embedDim + 2] = -R;
2330         coordsIn[4 * embedDim + 0] = -R;
2331         coordsIn[4 * embedDim + 1] = R;
2332         coordsIn[4 * embedDim + 2] = R;
2333         coordsIn[5 * embedDim + 0] = R;
2334         coordsIn[5 * embedDim + 1] = R;
2335         coordsIn[5 * embedDim + 2] = R;
2336         coordsIn[6 * embedDim + 0] = -R;
2337         coordsIn[6 * embedDim + 1] = -R;
2338         coordsIn[6 * embedDim + 2] = R;
2339         coordsIn[7 * embedDim + 0] = R;
2340         coordsIn[7 * embedDim + 1] = -R;
2341         coordsIn[7 * embedDim + 2] = R;
2342       }
2343     }
2344     break;
2345   case 3:
2346     if (simplex) {
2347       const PetscReal edgeLen         = 1.0 / PETSC_PHI;
2348       PetscReal       vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
2349       PetscReal       vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
2350       PetscReal       vertexC[4]      = {0.5, 0.5 * PETSC_PHI, 0.5 / PETSC_PHI, 0.0};
2351       const PetscInt  degree          = 12;
2352       PetscInt        s[4]            = {1, 1, 1};
2353       PetscInt        evenPerm[12][4] = {
2354         {0, 1, 2, 3},
2355         {0, 2, 3, 1},
2356         {0, 3, 1, 2},
2357         {1, 0, 3, 2},
2358         {1, 2, 0, 3},
2359         {1, 3, 2, 0},
2360         {2, 0, 1, 3},
2361         {2, 1, 3, 0},
2362         {2, 3, 0, 1},
2363         {3, 0, 2, 1},
2364         {3, 1, 0, 2},
2365         {3, 2, 1, 0}
2366       };
2367       PetscInt  cone[4];
2368       PetscInt *graph, p, i, j, k, l;
2369 
2370       vertexA[0] *= R;
2371       vertexA[1] *= R;
2372       vertexA[2] *= R;
2373       vertexA[3] *= R;
2374       vertexB[0] *= R;
2375       vertexB[1] *= R;
2376       vertexB[2] *= R;
2377       vertexB[3] *= R;
2378       vertexC[0] *= R;
2379       vertexC[1] *= R;
2380       vertexC[2] *= R;
2381       vertexC[3] *= R;
2382       numCells    = rank == 0 ? 600 : 0;
2383       numVerts    = rank == 0 ? 120 : 0;
2384       firstVertex = numCells;
2385       /* Use the 600-cell, which for a unit sphere has coordinates which are
2386 
2387            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
2388                (\pm 1,    0,       0,      0)  all cyclic permutations   8
2389            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
2390 
2391          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2392          length is then given by 1/\phi = 0.61803.
2393 
2394          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
2395          http://mathworld.wolfram.com/600-Cell.html
2396       */
2397       /* Construct vertices */
2398       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2399       i = 0;
2400       if (rank == 0) {
2401         for (s[0] = -1; s[0] < 2; s[0] += 2) {
2402           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2403             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2404               for (s[3] = -1; s[3] < 2; s[3] += 2) {
2405                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[d] * vertexA[d];
2406                 ++i;
2407               }
2408             }
2409           }
2410         }
2411         for (p = 0; p < embedDim; ++p) {
2412           s[1] = s[2] = s[3] = 1;
2413           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2414             for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertexB[(d + p) % embedDim];
2415             ++i;
2416           }
2417         }
2418         for (p = 0; p < 12; ++p) {
2419           s[3] = 1;
2420           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2421             for (s[1] = -1; s[1] < 2; s[1] += 2) {
2422               for (s[2] = -1; s[2] < 2; s[2] += 2) {
2423                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[evenPerm[p][d]] * vertexC[evenPerm[p][d]];
2424                 ++i;
2425               }
2426             }
2427           }
2428         }
2429       }
2430       PetscCheck(i == numVerts, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertices %" PetscInt_FMT " != %" PetscInt_FMT, i, numVerts);
2431       /* Construct graph */
2432       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2433       for (i = 0; i < numVerts; ++i) {
2434         for (j = 0, k = 0; j < numVerts; ++j) {
2435           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2436             graph[i * numVerts + j] = 1;
2437             ++k;
2438           }
2439         }
2440         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2441       }
2442       /* Build Topology */
2443       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2444       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2445       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2446       /* Cells */
2447       if (rank == 0) {
2448         for (i = 0, c = 0; i < numVerts; ++i) {
2449           for (j = 0; j < i; ++j) {
2450             for (k = 0; k < j; ++k) {
2451               for (l = 0; l < k; ++l) {
2452                 if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i] && graph[l * numVerts + i] && graph[l * numVerts + j] && graph[l * numVerts + k]) {
2453                   cone[0] = firstVertex + i;
2454                   cone[1] = firstVertex + j;
2455                   cone[2] = firstVertex + k;
2456                   cone[3] = firstVertex + l;
2457                   /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
2458                   {
2459                     const PetscInt epsilon[4][4][4][4] = {
2460                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, -1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 0, 0}, {0, 1, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 1, 0}, {0, -1, 0, 0}, {0, 0, 0, 0}}},
2461 
2462                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}}, {{0, 0, -1, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}}},
2463 
2464                       {{{0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, 0, 0}, {0, -1, 0, 0}}, {{0, 0, 0, -1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 1, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}},
2465 
2466                       {{{0, 0, 0, 0}, {0, 0, -1, 0}, {0, 1, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 1, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}}, {{0, -1, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}} }
2467                     };
2468                     PetscReal normal[4];
2469                     PetscInt  e, f, g;
2470 
2471                     for (d = 0; d < embedDim; ++d) {
2472                       normal[d] = 0.0;
2473                       for (e = 0; e < embedDim; ++e) {
2474                         for (f = 0; f < embedDim; ++f) {
2475                           for (g = 0; g < embedDim; ++g) {
2476                             normal[d] += epsilon[d][e][f][g] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]) * (coordsIn[l * embedDim + f] - coordsIn[i * embedDim + f]);
2477                           }
2478                         }
2479                       }
2480                     }
2481                     if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2482                       PetscInt tmp = cone[1];
2483                       cone[1]      = cone[2];
2484                       cone[2]      = tmp;
2485                     }
2486                   }
2487                   PetscCall(DMPlexSetCone(dm, c++, cone));
2488                 }
2489               }
2490             }
2491           }
2492         }
2493       }
2494       PetscCall(DMPlexSymmetrize(dm));
2495       PetscCall(DMPlexStratify(dm));
2496       PetscCall(PetscFree(graph));
2497     }
2498     break;
2499   default:
2500     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unsupported dimension for sphere: %" PetscInt_FMT, dim);
2501   }
2502   /* Create coordinates */
2503   PetscCall(DMGetCoordinateSection(dm, &coordSection));
2504   PetscCall(PetscSectionSetNumFields(coordSection, 1));
2505   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, embedDim));
2506   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVerts));
2507   for (v = firstVertex; v < firstVertex + numVerts; ++v) {
2508     PetscCall(PetscSectionSetDof(coordSection, v, embedDim));
2509     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, embedDim));
2510   }
2511   PetscCall(PetscSectionSetUp(coordSection));
2512   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2513   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2514   PetscCall(VecSetBlockSize(coordinates, embedDim));
2515   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2516   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2517   PetscCall(VecSetType(coordinates, VECSTANDARD));
2518   PetscCall(VecGetArray(coordinates, &coords));
2519   for (v = 0; v < numVerts; ++v)
2520     for (d = 0; d < embedDim; ++d) coords[v * embedDim + d] = coordsIn[v * embedDim + d];
2521   PetscCall(VecRestoreArray(coordinates, &coords));
2522   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2523   PetscCall(VecDestroy(&coordinates));
2524   PetscCall(PetscFree(coordsIn));
2525   {
2526     DM          cdm;
2527     PetscDS     cds;
2528     PetscScalar c = R;
2529 
2530     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToSphere));
2531     PetscCall(DMGetCoordinateDM(dm, &cdm));
2532     PetscCall(DMGetDS(cdm, &cds));
2533     PetscCall(PetscDSSetConstants(cds, 1, &c));
2534   }
2535   /* Wait for coordinate creation before doing in-place modification */
2536   if (simplex) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2537   PetscFunctionReturn(0);
2538 }
2539 
2540 typedef void (*TPSEvaluateFunc)(const PetscReal[], PetscReal *, PetscReal[], PetscReal (*)[3]);
2541 
2542 /*
2543  The Schwarz P implicit surface is
2544 
2545      f(x) = cos(x0) + cos(x1) + cos(x2) = 0
2546 */
2547 static void TPSEvaluate_SchwarzP(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2548 {
2549   PetscReal c[3] = {PetscCosReal(y[0] * PETSC_PI), PetscCosReal(y[1] * PETSC_PI), PetscCosReal(y[2] * PETSC_PI)};
2550   PetscReal g[3] = {-PetscSinReal(y[0] * PETSC_PI), -PetscSinReal(y[1] * PETSC_PI), -PetscSinReal(y[2] * PETSC_PI)};
2551   f[0]           = c[0] + c[1] + c[2];
2552   for (PetscInt i = 0; i < 3; i++) {
2553     grad[i] = PETSC_PI * g[i];
2554     for (PetscInt j = 0; j < 3; j++) hess[i][j] = (i == j) ? -PetscSqr(PETSC_PI) * c[i] : 0.;
2555   }
2556 }
2557 
2558 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2559 static PetscErrorCode TPSExtrudeNormalFunc_SchwarzP(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2560 {
2561   for (PetscInt i = 0; i < 3; i++) u[i] = -PETSC_PI * PetscSinReal(x[i] * PETSC_PI);
2562   return 0;
2563 }
2564 
2565 /*
2566  The Gyroid implicit surface is
2567 
2568  f(x,y,z) = sin(pi * x) * cos (pi * (y + 1/2))  + sin(pi * (y + 1/2)) * cos(pi * (z + 1/4)) + sin(pi * (z + 1/4)) * cos(pi * x)
2569 
2570 */
2571 static void TPSEvaluate_Gyroid(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2572 {
2573   PetscReal s[3] = {PetscSinReal(PETSC_PI * y[0]), PetscSinReal(PETSC_PI * (y[1] + .5)), PetscSinReal(PETSC_PI * (y[2] + .25))};
2574   PetscReal c[3] = {PetscCosReal(PETSC_PI * y[0]), PetscCosReal(PETSC_PI * (y[1] + .5)), PetscCosReal(PETSC_PI * (y[2] + .25))};
2575   f[0]           = s[0] * c[1] + s[1] * c[2] + s[2] * c[0];
2576   grad[0]        = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2577   grad[1]        = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2578   grad[2]        = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2579   hess[0][0]     = -PetscSqr(PETSC_PI) * (s[0] * c[1] + s[2] * c[0]);
2580   hess[0][1]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2581   hess[0][2]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2582   hess[1][0]     = -PetscSqr(PETSC_PI) * (s[1] * c[2] + s[0] * c[1]);
2583   hess[1][1]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2584   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2585   hess[2][0]     = -PetscSqr(PETSC_PI) * (s[2] * c[0] + s[1] * c[2]);
2586   hess[2][1]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2587   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2588 }
2589 
2590 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2591 static PetscErrorCode TPSExtrudeNormalFunc_Gyroid(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2592 {
2593   PetscReal s[3] = {PetscSinReal(PETSC_PI * x[0]), PetscSinReal(PETSC_PI * (x[1] + .5)), PetscSinReal(PETSC_PI * (x[2] + .25))};
2594   PetscReal c[3] = {PetscCosReal(PETSC_PI * x[0]), PetscCosReal(PETSC_PI * (x[1] + .5)), PetscCosReal(PETSC_PI * (x[2] + .25))};
2595   u[0]           = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2596   u[1]           = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2597   u[2]           = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2598   return 0;
2599 }
2600 
2601 /*
2602    We wish to solve
2603 
2604          min_y || y - x ||^2  subject to f(y) = 0
2605 
2606    Let g(y) = grad(f).  The minimization problem is equivalent to asking to satisfy
2607    f(y) = 0 and (y-x) is parallel to g(y).  We do this by using Householder QR to obtain a basis for the
2608    tangent space and ask for both components in the tangent space to be zero.
2609 
2610    Take g to be a column vector and compute the "full QR" factorization Q R = g,
2611    where Q = I - 2 n n^T is a symmetric orthogonal matrix.
2612    The first column of Q is parallel to g so the remaining two columns span the null space.
2613    Let Qn = Q[:,1:] be those remaining columns.  Then Qn Qn^T is an orthogonal projector into the tangent space.
2614    Since Q is symmetric, this is equivalent to multipyling by Q and taking the last two entries.
2615    In total, we have a system of 3 equations in 3 unknowns:
2616 
2617      f(y) = 0                       1 equation
2618      Qn^T (y - x) = 0               2 equations
2619 
2620    Here, we compute the residual and Jacobian of this system.
2621 */
2622 static void TPSNearestPointResJac(TPSEvaluateFunc feval, const PetscScalar x[], const PetscScalar y[], PetscScalar res[], PetscScalar J[])
2623 {
2624   PetscReal yreal[3] = {PetscRealPart(y[0]), PetscRealPart(y[1]), PetscRealPart(y[2])};
2625   PetscReal d[3]     = {PetscRealPart(y[0] - x[0]), PetscRealPart(y[1] - x[1]), PetscRealPart(y[2] - x[2])};
2626   PetscReal f, grad[3], n[3], norm, norm_y[3], nd, nd_y[3], sign;
2627   PetscReal n_y[3][3] = {
2628     {0, 0, 0},
2629     {0, 0, 0},
2630     {0, 0, 0}
2631   };
2632 
2633   feval(yreal, &f, grad, n_y);
2634 
2635   for (PetscInt i = 0; i < 3; i++) n[i] = grad[i];
2636   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2637   for (PetscInt i = 0; i < 3; i++) norm_y[i] = 1. / norm * n[i] * n_y[i][i];
2638 
2639   // Define the Householder reflector
2640   sign = n[0] >= 0 ? 1. : -1.;
2641   n[0] += norm * sign;
2642   for (PetscInt i = 0; i < 3; i++) n_y[0][i] += norm_y[i] * sign;
2643 
2644   norm      = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2645   norm_y[0] = 1. / norm * (n[0] * n_y[0][0]);
2646   norm_y[1] = 1. / norm * (n[0] * n_y[0][1] + n[1] * n_y[1][1]);
2647   norm_y[2] = 1. / norm * (n[0] * n_y[0][2] + n[2] * n_y[2][2]);
2648 
2649   for (PetscInt i = 0; i < 3; i++) {
2650     n[i] /= norm;
2651     for (PetscInt j = 0; j < 3; j++) {
2652       // note that n[i] is n_old[i]/norm when executing the code below
2653       n_y[i][j] = n_y[i][j] / norm - n[i] / norm * norm_y[j];
2654     }
2655   }
2656 
2657   nd = n[0] * d[0] + n[1] * d[1] + n[2] * d[2];
2658   for (PetscInt i = 0; i < 3; i++) nd_y[i] = n[i] + n_y[0][i] * d[0] + n_y[1][i] * d[1] + n_y[2][i] * d[2];
2659 
2660   res[0] = f;
2661   res[1] = d[1] - 2 * n[1] * nd;
2662   res[2] = d[2] - 2 * n[2] * nd;
2663   // J[j][i] is J_{ij} (column major)
2664   for (PetscInt j = 0; j < 3; j++) {
2665     J[0 + j * 3] = grad[j];
2666     J[1 + j * 3] = (j == 1) * 1. - 2 * (n_y[1][j] * nd + n[1] * nd_y[j]);
2667     J[2 + j * 3] = (j == 2) * 1. - 2 * (n_y[2][j] * nd + n[2] * nd_y[j]);
2668   }
2669 }
2670 
2671 /*
2672    Project x to the nearest point on the implicit surface using Newton's method.
2673 */
2674 static PetscErrorCode TPSNearestPoint(TPSEvaluateFunc feval, PetscScalar x[])
2675 {
2676   PetscScalar y[3] = {x[0], x[1], x[2]}; // Initial guess
2677 
2678   PetscFunctionBegin;
2679   for (PetscInt iter = 0; iter < 10; iter++) {
2680     PetscScalar res[3], J[9];
2681     PetscReal   resnorm;
2682     TPSNearestPointResJac(feval, x, y, res, J);
2683     resnorm = PetscSqrtReal(PetscSqr(PetscRealPart(res[0])) + PetscSqr(PetscRealPart(res[1])) + PetscSqr(PetscRealPart(res[2])));
2684     if (0) { // Turn on this monitor if you need to confirm quadratic convergence
2685       PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%" PetscInt_FMT "] res [%g %g %g]\n", iter, (double)PetscRealPart(res[0]), (double)PetscRealPart(res[1]), (double)PetscRealPart(res[2])));
2686     }
2687     if (resnorm < PETSC_SMALL) break;
2688 
2689     // Take the Newton step
2690     PetscCall(PetscKernel_A_gets_inverse_A_3(J, 0., PETSC_FALSE, NULL));
2691     PetscKernel_v_gets_v_minus_A_times_w_3(y, J, res);
2692   }
2693   for (PetscInt i = 0; i < 3; i++) x[i] = y[i];
2694   PetscFunctionReturn(0);
2695 }
2696 
2697 const char *const DMPlexTPSTypes[] = {"SCHWARZ_P", "GYROID", "DMPlexTPSType", "DMPLEX_TPS_", NULL};
2698 
2699 static PetscErrorCode DMPlexCreateTPSMesh_Internal(DM dm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness)
2700 {
2701   PetscMPIInt rank;
2702   PetscInt    topoDim = 2, spaceDim = 3, numFaces = 0, numVertices = 0, numEdges = 0;
2703   PetscInt(*edges)[2] = NULL, *edgeSets = NULL;
2704   PetscInt            *cells_flat = NULL;
2705   PetscReal           *vtxCoords  = NULL;
2706   TPSEvaluateFunc      evalFunc   = NULL;
2707   PetscSimplePointFunc normalFunc = NULL;
2708   DMLabel              label;
2709 
2710   PetscFunctionBegin;
2711   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2712   PetscCheck((layers != 0) ^ (thickness == 0.), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Layers %" PetscInt_FMT " must be nonzero iff thickness %g is nonzero", layers, (double)thickness);
2713   switch (tpstype) {
2714   case DMPLEX_TPS_SCHWARZ_P:
2715     PetscCheck(!periodic || (periodic[0] == DM_BOUNDARY_NONE && periodic[1] == DM_BOUNDARY_NONE && periodic[2] == DM_BOUNDARY_NONE), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Schwarz P does not support periodic meshes");
2716     if (rank == 0) {
2717       PetscInt(*cells)[6][4][4] = NULL; // [junction, junction-face, cell, conn]
2718       PetscInt  Njunctions = 0, Ncuts = 0, Npipes[3], vcount;
2719       PetscReal L = 1;
2720 
2721       Npipes[0]   = (extent[0] + 1) * extent[1] * extent[2];
2722       Npipes[1]   = extent[0] * (extent[1] + 1) * extent[2];
2723       Npipes[2]   = extent[0] * extent[1] * (extent[2] + 1);
2724       Njunctions  = extent[0] * extent[1] * extent[2];
2725       Ncuts       = 2 * (extent[0] * extent[1] + extent[1] * extent[2] + extent[2] * extent[0]);
2726       numVertices = 4 * (Npipes[0] + Npipes[1] + Npipes[2]) + 8 * Njunctions;
2727       PetscCall(PetscMalloc1(3 * numVertices, &vtxCoords));
2728       PetscCall(PetscMalloc1(Njunctions, &cells));
2729       PetscCall(PetscMalloc1(Ncuts * 4, &edges));
2730       PetscCall(PetscMalloc1(Ncuts * 4, &edgeSets));
2731       // x-normal pipes
2732       vcount = 0;
2733       for (PetscInt i = 0; i < extent[0] + 1; i++) {
2734         for (PetscInt j = 0; j < extent[1]; j++) {
2735           for (PetscInt k = 0; k < extent[2]; k++) {
2736             for (PetscInt l = 0; l < 4; l++) {
2737               vtxCoords[vcount++] = (2 * i - 1) * L;
2738               vtxCoords[vcount++] = 2 * j * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2739               vtxCoords[vcount++] = 2 * k * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2740             }
2741           }
2742         }
2743       }
2744       // y-normal pipes
2745       for (PetscInt i = 0; i < extent[0]; i++) {
2746         for (PetscInt j = 0; j < extent[1] + 1; j++) {
2747           for (PetscInt k = 0; k < extent[2]; k++) {
2748             for (PetscInt l = 0; l < 4; l++) {
2749               vtxCoords[vcount++] = 2 * i * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2750               vtxCoords[vcount++] = (2 * j - 1) * L;
2751               vtxCoords[vcount++] = 2 * k * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2752             }
2753           }
2754         }
2755       }
2756       // z-normal pipes
2757       for (PetscInt i = 0; i < extent[0]; i++) {
2758         for (PetscInt j = 0; j < extent[1]; j++) {
2759           for (PetscInt k = 0; k < extent[2] + 1; k++) {
2760             for (PetscInt l = 0; l < 4; l++) {
2761               vtxCoords[vcount++] = 2 * i * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2762               vtxCoords[vcount++] = 2 * j * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2763               vtxCoords[vcount++] = (2 * k - 1) * L;
2764             }
2765           }
2766         }
2767       }
2768       // junctions
2769       for (PetscInt i = 0; i < extent[0]; i++) {
2770         for (PetscInt j = 0; j < extent[1]; j++) {
2771           for (PetscInt k = 0; k < extent[2]; k++) {
2772             const PetscInt J = (i * extent[1] + j) * extent[2] + k, Jvoff = (Npipes[0] + Npipes[1] + Npipes[2]) * 4 + J * 8;
2773             PetscCheck(vcount / 3 == Jvoff, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected vertex count");
2774             for (PetscInt ii = 0; ii < 2; ii++) {
2775               for (PetscInt jj = 0; jj < 2; jj++) {
2776                 for (PetscInt kk = 0; kk < 2; kk++) {
2777                   double Ls           = (1 - sqrt(2) / 4) * L;
2778                   vtxCoords[vcount++] = 2 * i * L + (2 * ii - 1) * Ls;
2779                   vtxCoords[vcount++] = 2 * j * L + (2 * jj - 1) * Ls;
2780                   vtxCoords[vcount++] = 2 * k * L + (2 * kk - 1) * Ls;
2781                 }
2782               }
2783             }
2784             const PetscInt jfaces[3][2][4] = {
2785               {{3, 1, 0, 2}, {7, 5, 4, 6}}, // x-aligned
2786               {{5, 4, 0, 1}, {7, 6, 2, 3}}, // y-aligned
2787               {{6, 2, 0, 4}, {7, 3, 1, 5}}  // z-aligned
2788             };
2789             const PetscInt pipe_lo[3] = {// vertex numbers of pipes
2790                                          ((i * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + Npipes[0] + Npipes[1]) * 4};
2791             const PetscInt pipe_hi[3] = {// vertex numbers of pipes
2792                                          (((i + 1) * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j + 1) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + 1 + Npipes[0] + Npipes[1]) * 4};
2793             for (PetscInt dir = 0; dir < 3; dir++) { // x,y,z
2794               const PetscInt ijk[3] = {i, j, k};
2795               for (PetscInt l = 0; l < 4; l++) { // rotations
2796                 cells[J][dir * 2 + 0][l][0] = pipe_lo[dir] + l;
2797                 cells[J][dir * 2 + 0][l][1] = Jvoff + jfaces[dir][0][l];
2798                 cells[J][dir * 2 + 0][l][2] = Jvoff + jfaces[dir][0][(l - 1 + 4) % 4];
2799                 cells[J][dir * 2 + 0][l][3] = pipe_lo[dir] + (l - 1 + 4) % 4;
2800                 cells[J][dir * 2 + 1][l][0] = Jvoff + jfaces[dir][1][l];
2801                 cells[J][dir * 2 + 1][l][1] = pipe_hi[dir] + l;
2802                 cells[J][dir * 2 + 1][l][2] = pipe_hi[dir] + (l - 1 + 4) % 4;
2803                 cells[J][dir * 2 + 1][l][3] = Jvoff + jfaces[dir][1][(l - 1 + 4) % 4];
2804                 if (ijk[dir] == 0) {
2805                   edges[numEdges][0] = pipe_lo[dir] + l;
2806                   edges[numEdges][1] = pipe_lo[dir] + (l + 1) % 4;
2807                   edgeSets[numEdges] = dir * 2 + 1;
2808                   numEdges++;
2809                 }
2810                 if (ijk[dir] + 1 == extent[dir]) {
2811                   edges[numEdges][0] = pipe_hi[dir] + l;
2812                   edges[numEdges][1] = pipe_hi[dir] + (l + 1) % 4;
2813                   edgeSets[numEdges] = dir * 2 + 2;
2814                   numEdges++;
2815                 }
2816               }
2817             }
2818           }
2819         }
2820       }
2821       PetscCheck(numEdges == Ncuts * 4, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Edge count %" PetscInt_FMT " incompatible with number of cuts %" PetscInt_FMT, numEdges, Ncuts);
2822       numFaces   = 24 * Njunctions;
2823       cells_flat = cells[0][0][0];
2824     }
2825     evalFunc   = TPSEvaluate_SchwarzP;
2826     normalFunc = TPSExtrudeNormalFunc_SchwarzP;
2827     break;
2828   case DMPLEX_TPS_GYROID:
2829     if (rank == 0) {
2830       // This is a coarse mesh approximation of the gyroid shifted to being the zero of the level set
2831       //
2832       //     sin(pi*x)*cos(pi*(y+1/2)) + sin(pi*(y+1/2))*cos(pi*(z+1/4)) + sin(pi*(z+1/4))*cos(x)
2833       //
2834       // on the cell [0,2]^3.
2835       //
2836       // Think about dividing that cell into four columns, and focus on the column [0,1]x[0,1]x[0,2].
2837       // If you looked at the gyroid in that column at different slices of z you would see that it kind of spins
2838       // like a boomerang:
2839       //
2840       //     z = 0          z = 1/4        z = 1/2        z = 3/4     //
2841       //     -----          -------        -------        -------     //
2842       //                                                              //
2843       //     +       +      +       +      +       +      +   \   +   //
2844       //      \                                   /            \      //
2845       //       \            `-_   _-'            /              }     //
2846       //        *-_            `-'            _-'              /      //
2847       //     +     `-+      +       +      +-'     +      +   /   +   //
2848       //                                                              //
2849       //                                                              //
2850       //     z = 1          z = 5/4        z = 3/2        z = 7/4     //
2851       //     -----          -------        -------        -------     //
2852       //                                                              //
2853       //     +-_     +      +       +      +     _-+      +   /   +   //
2854       //        `-_            _-_            _-`            /        //
2855       //           \        _-'   `-_        /              {         //
2856       //            \                       /                \        //
2857       //     +       +      +       +      +       +      +   \   +   //
2858       //
2859       //
2860       // This course mesh approximates each of these slices by two line segments,
2861       // and then connects the segments in consecutive layers with quadrilateral faces.
2862       // All of the end points of the segments are multiples of 1/4 except for the
2863       // point * in the picture for z = 0 above and the similar points in other layers.
2864       // That point is at (gamma, gamma, 0), where gamma is calculated below.
2865       //
2866       // The column  [1,2]x[1,2]x[0,2] looks the same as this column;
2867       // The columns [1,2]x[0,1]x[0,2] and [0,1]x[1,2]x[0,2] are mirror images.
2868       //
2869       // As for how this method turned into the names given to the vertices:
2870       // that was not systematic, it was just the way it worked out in my handwritten notes.
2871 
2872       PetscInt facesPerBlock = 64;
2873       PetscInt vertsPerBlock = 56;
2874       PetscInt extentPlus[3];
2875       PetscInt numBlocks, numBlocksPlus;
2876       const PetscInt A = 0, B = 1, C = 2, D = 3, E = 4, F = 5, G = 6, H = 7, II = 8, J = 9, K = 10, L = 11, M = 12, N = 13, O = 14, P = 15, Q = 16, R = 17, S = 18, T = 19, U = 20, V = 21, W = 22, X = 23, Y = 24, Z = 25, Ap = 26, Bp = 27, Cp = 28, Dp = 29, Ep = 30, Fp = 31, Gp = 32, Hp = 33, Ip = 34, Jp = 35, Kp = 36, Lp = 37, Mp = 38, Np = 39, Op = 40, Pp = 41, Qp = 42, Rp = 43, Sp = 44, Tp = 45, Up = 46, Vp = 47, Wp = 48, Xp = 49, Yp = 50, Zp = 51, Aq = 52, Bq = 53, Cq = 54, Dq = 55;
2877       const PetscInt pattern[64][4] = {
2878   /* face to vertex within the coarse discretization of a single gyroid block */
2879   /* layer 0 */
2880         {A,           C,           K,           G          },
2881         {C,           B,           II,          K          },
2882         {D,           A,           H,           L          },
2883         {B + 56 * 1,  D,           L,           J          },
2884         {E,           B + 56 * 1,  J,           N          },
2885         {A + 56 * 2,  E,           N,           H + 56 * 2 },
2886         {F,           A + 56 * 2,  G + 56 * 2,  M          },
2887         {B,           F,           M,           II         },
2888  /* layer 1 */
2889         {G,           K,           Q,           O          },
2890         {K,           II,          P,           Q          },
2891         {L,           H,           O + 56 * 1,  R          },
2892         {J,           L,           R,           P          },
2893         {N,           J,           P,           S          },
2894         {H + 56 * 2,  N,           S,           O + 56 * 3 },
2895         {M,           G + 56 * 2,  O + 56 * 2,  T          },
2896         {II,          M,           T,           P          },
2897  /* layer 2 */
2898         {O,           Q,           Y,           U          },
2899         {Q,           P,           W,           Y          },
2900         {R,           O + 56 * 1,  U + 56 * 1,  Ap         },
2901         {P,           R,           Ap,          W          },
2902         {S,           P,           X,           Bp         },
2903         {O + 56 * 3,  S,           Bp,          V + 56 * 1 },
2904         {T,           O + 56 * 2,  V,           Z          },
2905         {P,           T,           Z,           X          },
2906  /* layer 3 */
2907         {U,           Y,           Ep,          Dp         },
2908         {Y,           W,           Cp,          Ep         },
2909         {Ap,          U + 56 * 1,  Dp + 56 * 1, Gp         },
2910         {W,           Ap,          Gp,          Cp         },
2911         {Bp,          X,           Cp + 56 * 2, Fp         },
2912         {V + 56 * 1,  Bp,          Fp,          Dp + 56 * 1},
2913         {Z,           V,           Dp,          Hp         },
2914         {X,           Z,           Hp,          Cp + 56 * 2},
2915  /* layer 4 */
2916         {Dp,          Ep,          Mp,          Kp         },
2917         {Ep,          Cp,          Ip,          Mp         },
2918         {Gp,          Dp + 56 * 1, Lp,          Np         },
2919         {Cp,          Gp,          Np,          Jp         },
2920         {Fp,          Cp + 56 * 2, Jp + 56 * 2, Pp         },
2921         {Dp + 56 * 1, Fp,          Pp,          Lp         },
2922         {Hp,          Dp,          Kp,          Op         },
2923         {Cp + 56 * 2, Hp,          Op,          Ip + 56 * 2},
2924  /* layer 5 */
2925         {Kp,          Mp,          Sp,          Rp         },
2926         {Mp,          Ip,          Qp,          Sp         },
2927         {Np,          Lp,          Rp,          Tp         },
2928         {Jp,          Np,          Tp,          Qp + 56 * 1},
2929         {Pp,          Jp + 56 * 2, Qp + 56 * 3, Up         },
2930         {Lp,          Pp,          Up,          Rp         },
2931         {Op,          Kp,          Rp,          Vp         },
2932         {Ip + 56 * 2, Op,          Vp,          Qp + 56 * 2},
2933  /* layer 6 */
2934         {Rp,          Sp,          Aq,          Yp         },
2935         {Sp,          Qp,          Wp,          Aq         },
2936         {Tp,          Rp,          Yp,          Cq         },
2937         {Qp + 56 * 1, Tp,          Cq,          Wp + 56 * 1},
2938         {Up,          Qp + 56 * 3, Xp + 56 * 1, Dq         },
2939         {Rp,          Up,          Dq,          Zp         },
2940         {Vp,          Rp,          Zp,          Bq         },
2941         {Qp + 56 * 2, Vp,          Bq,          Xp         },
2942  /* layer 7 (the top is the periodic image of the bottom of layer 0) */
2943         {Yp,          Aq,          C + 56 * 4,  A + 56 * 4 },
2944         {Aq,          Wp,          B + 56 * 4,  C + 56 * 4 },
2945         {Cq,          Yp,          A + 56 * 4,  D + 56 * 4 },
2946         {Wp + 56 * 1, Cq,          D + 56 * 4,  B + 56 * 5 },
2947         {Dq,          Xp + 56 * 1, B + 56 * 5,  E + 56 * 4 },
2948         {Zp,          Dq,          E + 56 * 4,  A + 56 * 6 },
2949         {Bq,          Zp,          A + 56 * 6,  F + 56 * 4 },
2950         {Xp,          Bq,          F + 56 * 4,  B + 56 * 4 }
2951       };
2952       const PetscReal gamma                = PetscAcosReal((PetscSqrtReal(3.) - 1.) / PetscSqrtReal(2.)) / PETSC_PI;
2953       const PetscReal patternCoords[56][3] = {
2954         {1.,        0.,        0.  }, /* A  */
2955         {0.,        1.,        0.  }, /* B  */
2956         {gamma,     gamma,     0.  }, /* C  */
2957         {1 + gamma, 1 - gamma, 0.  }, /* D  */
2958         {2 - gamma, 2 - gamma, 0.  }, /* E  */
2959         {1 - gamma, 1 + gamma, 0.  }, /* F  */
2960 
2961         {.5,        0,         .25 }, /* G  */
2962         {1.5,       0.,        .25 }, /* H  */
2963         {.5,        1.,        .25 }, /* II */
2964         {1.5,       1.,        .25 }, /* J  */
2965         {.25,       .5,        .25 }, /* K  */
2966         {1.25,      .5,        .25 }, /* L  */
2967         {.75,       1.5,       .25 }, /* M  */
2968         {1.75,      1.5,       .25 }, /* N  */
2969 
2970         {0.,        0.,        .5  }, /* O  */
2971         {1.,        1.,        .5  }, /* P  */
2972         {gamma,     1 - gamma, .5  }, /* Q  */
2973         {1 + gamma, gamma,     .5  }, /* R  */
2974         {2 - gamma, 1 + gamma, .5  }, /* S  */
2975         {1 - gamma, 2 - gamma, .5  }, /* T  */
2976 
2977         {0.,        .5,        .75 }, /* U  */
2978         {0.,        1.5,       .75 }, /* V  */
2979         {1.,        .5,        .75 }, /* W  */
2980         {1.,        1.5,       .75 }, /* X  */
2981         {.5,        .75,       .75 }, /* Y  */
2982         {.5,        1.75,      .75 }, /* Z  */
2983         {1.5,       .25,       .75 }, /* Ap */
2984         {1.5,       1.25,      .75 }, /* Bp */
2985 
2986         {1.,        0.,        1.  }, /* Cp */
2987         {0.,        1.,        1.  }, /* Dp */
2988         {1 - gamma, 1 - gamma, 1.  }, /* Ep */
2989         {1 + gamma, 1 + gamma, 1.  }, /* Fp */
2990         {2 - gamma, gamma,     1.  }, /* Gp */
2991         {gamma,     2 - gamma, 1.  }, /* Hp */
2992 
2993         {.5,        0.,        1.25}, /* Ip */
2994         {1.5,       0.,        1.25}, /* Jp */
2995         {.5,        1.,        1.25}, /* Kp */
2996         {1.5,       1.,        1.25}, /* Lp */
2997         {.75,       .5,        1.25}, /* Mp */
2998         {1.75,      .5,        1.25}, /* Np */
2999         {.25,       1.5,       1.25}, /* Op */
3000         {1.25,      1.5,       1.25}, /* Pp */
3001 
3002         {0.,        0.,        1.5 }, /* Qp */
3003         {1.,        1.,        1.5 }, /* Rp */
3004         {1 - gamma, gamma,     1.5 }, /* Sp */
3005         {2 - gamma, 1 - gamma, 1.5 }, /* Tp */
3006         {1 + gamma, 2 - gamma, 1.5 }, /* Up */
3007         {gamma,     1 + gamma, 1.5 }, /* Vp */
3008 
3009         {0.,        .5,        1.75}, /* Wp */
3010         {0.,        1.5,       1.75}, /* Xp */
3011         {1.,        .5,        1.75}, /* Yp */
3012         {1.,        1.5,       1.75}, /* Zp */
3013         {.5,        .25,       1.75}, /* Aq */
3014         {.5,        1.25,      1.75}, /* Bq */
3015         {1.5,       .75,       1.75}, /* Cq */
3016         {1.5,       1.75,      1.75}, /* Dq */
3017       };
3018       PetscInt(*cells)[64][4] = NULL;
3019       PetscBool *seen;
3020       PetscInt  *vertToTrueVert;
3021       PetscInt   count;
3022 
3023       for (PetscInt i = 0; i < 3; i++) extentPlus[i] = extent[i] + 1;
3024       numBlocks = 1;
3025       for (PetscInt i = 0; i < 3; i++) numBlocks *= extent[i];
3026       numBlocksPlus = 1;
3027       for (PetscInt i = 0; i < 3; i++) numBlocksPlus *= extentPlus[i];
3028       numFaces = numBlocks * facesPerBlock;
3029       PetscCall(PetscMalloc1(numBlocks, &cells));
3030       PetscCall(PetscCalloc1(numBlocksPlus * vertsPerBlock, &seen));
3031       for (PetscInt k = 0; k < extent[2]; k++) {
3032         for (PetscInt j = 0; j < extent[1]; j++) {
3033           for (PetscInt i = 0; i < extent[0]; i++) {
3034             for (PetscInt f = 0; f < facesPerBlock; f++) {
3035               for (PetscInt v = 0; v < 4; v++) {
3036                 PetscInt vertRaw     = pattern[f][v];
3037                 PetscInt blockidx    = vertRaw / 56;
3038                 PetscInt patternvert = vertRaw % 56;
3039                 PetscInt xplus       = (blockidx & 1);
3040                 PetscInt yplus       = (blockidx & 2) >> 1;
3041                 PetscInt zplus       = (blockidx & 4) >> 2;
3042                 PetscInt zcoord      = (periodic && periodic[2] == DM_BOUNDARY_PERIODIC) ? ((k + zplus) % extent[2]) : (k + zplus);
3043                 PetscInt ycoord      = (periodic && periodic[1] == DM_BOUNDARY_PERIODIC) ? ((j + yplus) % extent[1]) : (j + yplus);
3044                 PetscInt xcoord      = (periodic && periodic[0] == DM_BOUNDARY_PERIODIC) ? ((i + xplus) % extent[0]) : (i + xplus);
3045                 PetscInt vert        = ((zcoord * extentPlus[1] + ycoord) * extentPlus[0] + xcoord) * 56 + patternvert;
3046 
3047                 cells[(k * extent[1] + j) * extent[0] + i][f][v] = vert;
3048                 seen[vert]                                       = PETSC_TRUE;
3049               }
3050             }
3051           }
3052         }
3053       }
3054       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++)
3055         if (seen[i]) numVertices++;
3056       count = 0;
3057       PetscCall(PetscMalloc1(numBlocksPlus * vertsPerBlock, &vertToTrueVert));
3058       PetscCall(PetscMalloc1(numVertices * 3, &vtxCoords));
3059       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) vertToTrueVert[i] = -1;
3060       for (PetscInt k = 0; k < extentPlus[2]; k++) {
3061         for (PetscInt j = 0; j < extentPlus[1]; j++) {
3062           for (PetscInt i = 0; i < extentPlus[0]; i++) {
3063             for (PetscInt v = 0; v < vertsPerBlock; v++) {
3064               PetscInt vIdx = ((k * extentPlus[1] + j) * extentPlus[0] + i) * vertsPerBlock + v;
3065 
3066               if (seen[vIdx]) {
3067                 PetscInt thisVert;
3068 
3069                 vertToTrueVert[vIdx] = thisVert = count++;
3070 
3071                 for (PetscInt d = 0; d < 3; d++) vtxCoords[3 * thisVert + d] = patternCoords[v][d];
3072                 vtxCoords[3 * thisVert + 0] += i * 2;
3073                 vtxCoords[3 * thisVert + 1] += j * 2;
3074                 vtxCoords[3 * thisVert + 2] += k * 2;
3075               }
3076             }
3077           }
3078         }
3079       }
3080       for (PetscInt i = 0; i < numBlocks; i++) {
3081         for (PetscInt f = 0; f < facesPerBlock; f++) {
3082           for (PetscInt v = 0; v < 4; v++) cells[i][f][v] = vertToTrueVert[cells[i][f][v]];
3083         }
3084       }
3085       PetscCall(PetscFree(vertToTrueVert));
3086       PetscCall(PetscFree(seen));
3087       cells_flat = cells[0][0];
3088       numEdges   = 0;
3089       for (PetscInt i = 0; i < numFaces; i++) {
3090         for (PetscInt e = 0; e < 4; e++) {
3091           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3092           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3093 
3094           for (PetscInt d = 0; d < 3; d++) {
3095             if (!periodic || periodic[0] != DM_BOUNDARY_PERIODIC) {
3096               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) numEdges++;
3097               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) numEdges++;
3098             }
3099           }
3100         }
3101       }
3102       PetscCall(PetscMalloc1(numEdges, &edges));
3103       PetscCall(PetscMalloc1(numEdges, &edgeSets));
3104       for (PetscInt edge = 0, i = 0; i < numFaces; i++) {
3105         for (PetscInt e = 0; e < 4; e++) {
3106           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3107           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3108 
3109           for (PetscInt d = 0; d < 3; d++) {
3110             if (!periodic || periodic[d] != DM_BOUNDARY_PERIODIC) {
3111               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) {
3112                 edges[edge][0]   = ev[0];
3113                 edges[edge][1]   = ev[1];
3114                 edgeSets[edge++] = 2 * d;
3115               }
3116               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) {
3117                 edges[edge][0]   = ev[0];
3118                 edges[edge][1]   = ev[1];
3119                 edgeSets[edge++] = 2 * d + 1;
3120               }
3121             }
3122           }
3123         }
3124       }
3125     }
3126     evalFunc   = TPSEvaluate_Gyroid;
3127     normalFunc = TPSExtrudeNormalFunc_Gyroid;
3128     break;
3129   }
3130 
3131   PetscCall(DMSetDimension(dm, topoDim));
3132   if (rank == 0) PetscCall(DMPlexBuildFromCellList(dm, numFaces, numVertices, 4, cells_flat));
3133   else PetscCall(DMPlexBuildFromCellList(dm, 0, 0, 0, NULL));
3134   PetscCall(PetscFree(cells_flat));
3135   {
3136     DM idm;
3137     PetscCall(DMPlexInterpolate(dm, &idm));
3138     PetscCall(DMPlexReplace_Internal(dm, &idm));
3139   }
3140   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, vtxCoords));
3141   else PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, NULL));
3142   PetscCall(PetscFree(vtxCoords));
3143 
3144   PetscCall(DMCreateLabel(dm, "Face Sets"));
3145   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3146   for (PetscInt e = 0; e < numEdges; e++) {
3147     PetscInt        njoin;
3148     const PetscInt *join, verts[] = {numFaces + edges[e][0], numFaces + edges[e][1]};
3149     PetscCall(DMPlexGetJoin(dm, 2, verts, &njoin, &join));
3150     PetscCheck(njoin == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected unique join of vertices %" PetscInt_FMT " and %" PetscInt_FMT, edges[e][0], edges[e][1]);
3151     PetscCall(DMLabelSetValue(label, join[0], edgeSets[e]));
3152     PetscCall(DMPlexRestoreJoin(dm, 2, verts, &njoin, &join));
3153   }
3154   PetscCall(PetscFree(edges));
3155   PetscCall(PetscFree(edgeSets));
3156   if (tps_distribute) {
3157     DM               pdm = NULL;
3158     PetscPartitioner part;
3159 
3160     PetscCall(DMPlexGetPartitioner(dm, &part));
3161     PetscCall(PetscPartitionerSetFromOptions(part));
3162     PetscCall(DMPlexDistribute(dm, 0, NULL, &pdm));
3163     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
3164     // Do not auto-distribute again
3165     PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
3166   }
3167 
3168   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3169   for (PetscInt refine = 0; refine < refinements; refine++) {
3170     PetscInt     m;
3171     DM           dmf;
3172     Vec          X;
3173     PetscScalar *x;
3174     PetscCall(DMRefine(dm, MPI_COMM_NULL, &dmf));
3175     PetscCall(DMPlexReplace_Internal(dm, &dmf));
3176 
3177     PetscCall(DMGetCoordinatesLocal(dm, &X));
3178     PetscCall(VecGetLocalSize(X, &m));
3179     PetscCall(VecGetArray(X, &x));
3180     for (PetscInt i = 0; i < m; i += 3) PetscCall(TPSNearestPoint(evalFunc, &x[i]));
3181     PetscCall(VecRestoreArray(X, &x));
3182   }
3183 
3184   // Face Sets has already been propagated to new vertices during refinement; this propagates to the initial vertices.
3185   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3186   PetscCall(DMPlexLabelComplete(dm, label));
3187 
3188   if (thickness > 0) {
3189     DM              edm, cdm, ecdm;
3190     DMPlexTransform tr;
3191     const char     *prefix;
3192     PetscOptions    options;
3193     // Code from DMPlexExtrude
3194     PetscCall(DMPlexTransformCreate(PetscObjectComm((PetscObject)dm), &tr));
3195     PetscCall(DMPlexTransformSetDM(tr, dm));
3196     PetscCall(DMPlexTransformSetType(tr, DMPLEXEXTRUDE));
3197     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
3198     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)tr, prefix));
3199     PetscCall(PetscObjectGetOptions((PetscObject)dm, &options));
3200     PetscCall(PetscObjectSetOptions((PetscObject)tr, options));
3201     PetscCall(DMPlexTransformExtrudeSetLayers(tr, layers));
3202     PetscCall(DMPlexTransformExtrudeSetThickness(tr, thickness));
3203     PetscCall(DMPlexTransformExtrudeSetTensor(tr, PETSC_FALSE));
3204     PetscCall(DMPlexTransformExtrudeSetSymmetric(tr, PETSC_TRUE));
3205     PetscCall(DMPlexTransformExtrudeSetNormalFunction(tr, normalFunc));
3206     PetscCall(DMPlexTransformSetFromOptions(tr));
3207     PetscCall(PetscObjectSetOptions((PetscObject)tr, NULL));
3208     PetscCall(DMPlexTransformSetUp(tr));
3209     PetscCall(PetscObjectViewFromOptions((PetscObject)tr, NULL, "-dm_plex_tps_transform_view"));
3210     PetscCall(DMPlexTransformApply(tr, dm, &edm));
3211     PetscCall(DMCopyDisc(dm, edm));
3212     PetscCall(DMGetCoordinateDM(dm, &cdm));
3213     PetscCall(DMGetCoordinateDM(edm, &ecdm));
3214     PetscCall(DMCopyDisc(cdm, ecdm));
3215     PetscCall(DMPlexTransformCreateDiscLabels(tr, edm));
3216     PetscCall(DMPlexTransformDestroy(&tr));
3217     if (edm) {
3218       ((DM_Plex *)edm->data)->printFEM    = ((DM_Plex *)dm->data)->printFEM;
3219       ((DM_Plex *)edm->data)->printL2     = ((DM_Plex *)dm->data)->printL2;
3220       ((DM_Plex *)edm->data)->printLocate = ((DM_Plex *)dm->data)->printLocate;
3221     }
3222     PetscCall(DMPlexReplace_Internal(dm, &edm));
3223   }
3224   PetscFunctionReturn(0);
3225 }
3226 
3227 /*@
3228   DMPlexCreateTPSMesh - Create a distributed, interpolated mesh of a triply-periodic surface
3229 
3230   Collective
3231 
3232   Input Parameters:
3233 + comm   - The communicator for the DM object
3234 . tpstype - Type of triply-periodic surface
3235 . extent - Array of length 3 containing number of periods in each direction
3236 . periodic - array of length 3 with periodicity, or NULL for non-periodic
3237 . tps_distribute - Distribute 2D manifold mesh prior to refinement and extrusion (more scalable)
3238 . refinements - Number of factor-of-2 refinements of 2D manifold mesh
3239 . layers - Number of cell layers extruded in normal direction
3240 - thickness - Thickness in normal direction
3241 
3242   Output Parameter:
3243 . dm  - The DM object
3244 
3245   Notes:
3246   This meshes the surface of the Schwarz P or Gyroid surfaces.  Schwarz P is is the simplest member of the triply-periodic minimal surfaces.
3247   https://en.wikipedia.org/wiki/Schwarz_minimal_surface#Schwarz_P_(%22Primitive%22) and can be cut with "clean" boundaries.
3248   The Gyroid (https://en.wikipedia.org/wiki/Gyroid) is another triply-periodic minimal surface with applications in additive manufacturing; it is much more difficult to "cut" since there are no planes of symmetry.
3249   Our implementation creates a very coarse mesh of the surface and refines (by 4-way splitting) as many times as requested.
3250   On each refinement, all vertices are projected to their nearest point on the surface.
3251   This projection could readily be extended to related surfaces.
3252 
3253   The face (edge) sets for the Schwarz P surface are numbered 1(-x), 2(+x), 3(-y), 4(+y), 5(-z), 6(+z).
3254   When the mesh is refined, "Face Sets" contain the new vertices (created during refinement).  Use DMPlexLabelComplete() to propagate to coarse-level vertices.
3255 
3256   References:
3257 . * - Maskery et al, Insights into the mechanical properties of several triply periodic minimal surface lattice structures made by polymer additive manufacturing, 2017. https://doi.org/10.1016/j.polymer.2017.11.049
3258 
3259   Developer Notes:
3260   The Gyroid mesh does not currently mark boundary sets.
3261 
3262   Level: beginner
3263 
3264 .seealso: `DMPlexCreateSphereMesh()`, `DMSetType()`, `DMCreate()`
3265 @*/
3266 PetscErrorCode DMPlexCreateTPSMesh(MPI_Comm comm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness, DM *dm)
3267 {
3268   PetscFunctionBegin;
3269   PetscCall(DMCreate(comm, dm));
3270   PetscCall(DMSetType(*dm, DMPLEX));
3271   PetscCall(DMPlexCreateTPSMesh_Internal(*dm, tpstype, extent, periodic, tps_distribute, refinements, layers, thickness));
3272   PetscFunctionReturn(0);
3273 }
3274 
3275 /*@
3276   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
3277 
3278   Collective
3279 
3280   Input Parameters:
3281 + comm    - The communicator for the DM object
3282 . dim     - The dimension
3283 . simplex - Use simplices, or tensor product cells
3284 - R       - The radius
3285 
3286   Output Parameter:
3287 . dm  - The DM object
3288 
3289   Level: beginner
3290 
3291 .seealso: `DMPlexCreateBallMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3292 @*/
3293 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscReal R, DM *dm)
3294 {
3295   PetscFunctionBegin;
3296   PetscValidPointer(dm, 5);
3297   PetscCall(DMCreate(comm, dm));
3298   PetscCall(DMSetType(*dm, DMPLEX));
3299   PetscCall(DMPlexCreateSphereMesh_Internal(*dm, dim, simplex, R));
3300   PetscFunctionReturn(0);
3301 }
3302 
3303 static PetscErrorCode DMPlexCreateBallMesh_Internal(DM dm, PetscInt dim, PetscReal R)
3304 {
3305   DM      sdm, vol;
3306   DMLabel bdlabel;
3307 
3308   PetscFunctionBegin;
3309   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
3310   PetscCall(DMSetType(sdm, DMPLEX));
3311   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sdm, "bd_"));
3312   PetscCall(DMPlexCreateSphereMesh_Internal(sdm, dim - 1, PETSC_TRUE, R));
3313   PetscCall(DMSetFromOptions(sdm));
3314   PetscCall(DMViewFromOptions(sdm, NULL, "-dm_view"));
3315   PetscCall(DMPlexGenerate(sdm, NULL, PETSC_TRUE, &vol));
3316   PetscCall(DMDestroy(&sdm));
3317   PetscCall(DMPlexReplace_Internal(dm, &vol));
3318   PetscCall(DMCreateLabel(dm, "marker"));
3319   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
3320   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
3321   PetscCall(DMPlexLabelComplete(dm, bdlabel));
3322   PetscFunctionReturn(0);
3323 }
3324 
3325 /*@
3326   DMPlexCreateBallMesh - Creates a simplex mesh on the d-dimensional ball, B^d.
3327 
3328   Collective
3329 
3330   Input Parameters:
3331 + comm  - The communicator for the DM object
3332 . dim   - The dimension
3333 - R     - The radius
3334 
3335   Output Parameter:
3336 . dm  - The DM object
3337 
3338   Options Database Keys:
3339 - bd_dm_refine - This will refine the surface mesh preserving the sphere geometry
3340 
3341   Level: beginner
3342 
3343 .seealso: `DMPlexCreateSphereMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3344 @*/
3345 PetscErrorCode DMPlexCreateBallMesh(MPI_Comm comm, PetscInt dim, PetscReal R, DM *dm)
3346 {
3347   PetscFunctionBegin;
3348   PetscCall(DMCreate(comm, dm));
3349   PetscCall(DMSetType(*dm, DMPLEX));
3350   PetscCall(DMPlexCreateBallMesh_Internal(*dm, dim, R));
3351   PetscFunctionReturn(0);
3352 }
3353 
3354 static PetscErrorCode DMPlexCreateReferenceCell_Internal(DM rdm, DMPolytopeType ct)
3355 {
3356   PetscFunctionBegin;
3357   switch (ct) {
3358   case DM_POLYTOPE_POINT: {
3359     PetscInt    numPoints[1]        = {1};
3360     PetscInt    coneSize[1]         = {0};
3361     PetscInt    cones[1]            = {0};
3362     PetscInt    coneOrientations[1] = {0};
3363     PetscScalar vertexCoords[1]     = {0.0};
3364 
3365     PetscCall(DMSetDimension(rdm, 0));
3366     PetscCall(DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3367   } break;
3368   case DM_POLYTOPE_SEGMENT: {
3369     PetscInt    numPoints[2]        = {2, 1};
3370     PetscInt    coneSize[3]         = {2, 0, 0};
3371     PetscInt    cones[2]            = {1, 2};
3372     PetscInt    coneOrientations[2] = {0, 0};
3373     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3374 
3375     PetscCall(DMSetDimension(rdm, 1));
3376     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3377   } break;
3378   case DM_POLYTOPE_POINT_PRISM_TENSOR: {
3379     PetscInt    numPoints[2]        = {2, 1};
3380     PetscInt    coneSize[3]         = {2, 0, 0};
3381     PetscInt    cones[2]            = {1, 2};
3382     PetscInt    coneOrientations[2] = {0, 0};
3383     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3384 
3385     PetscCall(DMSetDimension(rdm, 1));
3386     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3387   } break;
3388   case DM_POLYTOPE_TRIANGLE: {
3389     PetscInt    numPoints[2]        = {3, 1};
3390     PetscInt    coneSize[4]         = {3, 0, 0, 0};
3391     PetscInt    cones[3]            = {1, 2, 3};
3392     PetscInt    coneOrientations[3] = {0, 0, 0};
3393     PetscScalar vertexCoords[6]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0};
3394 
3395     PetscCall(DMSetDimension(rdm, 2));
3396     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3397   } break;
3398   case DM_POLYTOPE_QUADRILATERAL: {
3399     PetscInt    numPoints[2]        = {4, 1};
3400     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3401     PetscInt    cones[4]            = {1, 2, 3, 4};
3402     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3403     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0};
3404 
3405     PetscCall(DMSetDimension(rdm, 2));
3406     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3407   } break;
3408   case DM_POLYTOPE_SEG_PRISM_TENSOR: {
3409     PetscInt    numPoints[2]        = {4, 1};
3410     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3411     PetscInt    cones[4]            = {1, 2, 3, 4};
3412     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3413     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0};
3414 
3415     PetscCall(DMSetDimension(rdm, 2));
3416     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3417   } break;
3418   case DM_POLYTOPE_TETRAHEDRON: {
3419     PetscInt    numPoints[2]        = {4, 1};
3420     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3421     PetscInt    cones[4]            = {1, 2, 3, 4};
3422     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3423     PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0};
3424 
3425     PetscCall(DMSetDimension(rdm, 3));
3426     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3427   } break;
3428   case DM_POLYTOPE_HEXAHEDRON: {
3429     PetscInt    numPoints[2]        = {8, 1};
3430     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3431     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3432     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3433     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3434 
3435     PetscCall(DMSetDimension(rdm, 3));
3436     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3437   } break;
3438   case DM_POLYTOPE_TRI_PRISM: {
3439     PetscInt    numPoints[2]        = {6, 1};
3440     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3441     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3442     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3443     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3444 
3445     PetscCall(DMSetDimension(rdm, 3));
3446     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3447   } break;
3448   case DM_POLYTOPE_TRI_PRISM_TENSOR: {
3449     PetscInt    numPoints[2]        = {6, 1};
3450     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3451     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3452     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3453     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3454 
3455     PetscCall(DMSetDimension(rdm, 3));
3456     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3457   } break;
3458   case DM_POLYTOPE_QUAD_PRISM_TENSOR: {
3459     PetscInt    numPoints[2]        = {8, 1};
3460     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3461     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3462     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3463     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3464 
3465     PetscCall(DMSetDimension(rdm, 3));
3466     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3467   } break;
3468   case DM_POLYTOPE_PYRAMID: {
3469     PetscInt    numPoints[2]        = {5, 1};
3470     PetscInt    coneSize[6]         = {5, 0, 0, 0, 0, 0};
3471     PetscInt    cones[5]            = {1, 2, 3, 4, 5};
3472     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3473     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, 0.0, 0.0, 1.0};
3474 
3475     PetscCall(DMSetDimension(rdm, 3));
3476     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3477   } break;
3478   default:
3479     SETERRQ(PetscObjectComm((PetscObject)rdm), PETSC_ERR_ARG_WRONG, "Cannot create reference cell for cell type %s", DMPolytopeTypes[ct]);
3480   }
3481   {
3482     PetscInt Nv, v;
3483 
3484     /* Must create the celltype label here so that we do not automatically try to compute the types */
3485     PetscCall(DMCreateLabel(rdm, "celltype"));
3486     PetscCall(DMPlexSetCellType(rdm, 0, ct));
3487     PetscCall(DMPlexGetChart(rdm, NULL, &Nv));
3488     for (v = 1; v < Nv; ++v) PetscCall(DMPlexSetCellType(rdm, v, DM_POLYTOPE_POINT));
3489   }
3490   PetscCall(DMPlexInterpolateInPlace_Internal(rdm));
3491   PetscCall(PetscObjectSetName((PetscObject)rdm, DMPolytopeTypes[ct]));
3492   PetscFunctionReturn(0);
3493 }
3494 
3495 /*@
3496   DMPlexCreateReferenceCell - Create a DMPLEX with the appropriate FEM reference cell
3497 
3498   Collective
3499 
3500   Input Parameters:
3501 + comm - The communicator
3502 - ct   - The cell type of the reference cell
3503 
3504   Output Parameter:
3505 . refdm - The reference cell
3506 
3507   Level: intermediate
3508 
3509 .seealso: `DMPlexCreateReferenceCell()`, `DMPlexCreateBoxMesh()`
3510 @*/
3511 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, DMPolytopeType ct, DM *refdm)
3512 {
3513   PetscFunctionBegin;
3514   PetscCall(DMCreate(comm, refdm));
3515   PetscCall(DMSetType(*refdm, DMPLEX));
3516   PetscCall(DMPlexCreateReferenceCell_Internal(*refdm, ct));
3517   PetscFunctionReturn(0);
3518 }
3519 
3520 static PetscErrorCode DMPlexCreateBoundaryLabel_Private(DM dm, const char name[])
3521 {
3522   DM        plex;
3523   DMLabel   label;
3524   PetscBool hasLabel;
3525 
3526   PetscFunctionBegin;
3527   PetscCall(DMHasLabel(dm, name, &hasLabel));
3528   if (hasLabel) PetscFunctionReturn(0);
3529   PetscCall(DMCreateLabel(dm, name));
3530   PetscCall(DMGetLabel(dm, name, &label));
3531   PetscCall(DMConvert(dm, DMPLEX, &plex));
3532   PetscCall(DMPlexMarkBoundaryFaces(plex, 1, label));
3533   PetscCall(DMPlexLabelComplete(plex, label));
3534   PetscCall(DMDestroy(&plex));
3535   PetscFunctionReturn(0);
3536 }
3537 
3538 /*
3539   We use the last coordinate as the radius, the inner radius is lower[dim-1] and the outer radius is upper[dim-1]. Then we map the first coordinate around the circle.
3540 
3541     (x, y) -> (r, theta) = (x[1], (x[0] - lower[0]) * 2\pi/(upper[0] - lower[0]))
3542 */
3543 static void boxToAnnulus(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
3544 {
3545   const PetscReal low = PetscRealPart(constants[0]);
3546   const PetscReal upp = PetscRealPart(constants[1]);
3547   const PetscReal r   = PetscRealPart(u[1]);
3548   const PetscReal th  = 2. * PETSC_PI * (PetscRealPart(u[0]) - low) / (upp - low);
3549 
3550   f0[0] = r * PetscCosReal(th);
3551   f0[1] = r * PetscSinReal(th);
3552 }
3553 
3554 const char *const DMPlexShapes[] = {"box", "box_surface", "ball", "sphere", "cylinder", "schwarz_p", "gyroid", "doublet", "annulus", "unknown", "DMPlexShape", "DM_SHAPE_", NULL};
3555 
3556 static PetscErrorCode DMPlexCreateFromOptions_Internal(PetscOptionItems *PetscOptionsObject, PetscBool *useCoordSpace, DM dm)
3557 {
3558   DMPlexShape    shape   = DM_SHAPE_BOX;
3559   DMPolytopeType cell    = DM_POLYTOPE_TRIANGLE;
3560   PetscInt       dim     = 2;
3561   PetscBool      simplex = PETSC_TRUE, interpolate = PETSC_TRUE, adjCone = PETSC_FALSE, adjClosure = PETSC_TRUE, refDomain = PETSC_FALSE;
3562   PetscBool      flg, flg2, fflg, bdfflg, nameflg;
3563   MPI_Comm       comm;
3564   char           filename[PETSC_MAX_PATH_LEN]   = "<unspecified>";
3565   char           bdFilename[PETSC_MAX_PATH_LEN] = "<unspecified>";
3566   char           plexname[PETSC_MAX_PATH_LEN]   = "";
3567 
3568   PetscFunctionBegin;
3569   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3570   /* TODO Turn this into a registration interface */
3571   PetscCall(PetscOptionsString("-dm_plex_filename", "File containing a mesh", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &fflg));
3572   PetscCall(PetscOptionsString("-dm_plex_boundary_filename", "File containing a mesh boundary", "DMPlexCreateFromFile", bdFilename, bdFilename, sizeof(bdFilename), &bdfflg));
3573   PetscCall(PetscOptionsString("-dm_plex_name", "Name of the mesh in the file", "DMPlexCreateFromFile", plexname, plexname, sizeof(plexname), &nameflg));
3574   PetscCall(PetscOptionsEnum("-dm_plex_cell", "Cell shape", "", DMPolytopeTypes, (PetscEnum)cell, (PetscEnum *)&cell, NULL));
3575   PetscCall(PetscOptionsBool("-dm_plex_reference_cell_domain", "Use a reference cell domain", "", refDomain, &refDomain, NULL));
3576   PetscCall(PetscOptionsEnum("-dm_plex_shape", "Shape for built-in mesh", "", DMPlexShapes, (PetscEnum)shape, (PetscEnum *)&shape, &flg));
3577   PetscCall(PetscOptionsBoundedInt("-dm_plex_dim", "Topological dimension of the mesh", "DMGetDimension", dim, &dim, &flg, 0));
3578   PetscCheck(!(dim < 0) && !(dim > 3), comm, PETSC_ERR_ARG_OUTOFRANGE, "Dimension %" PetscInt_FMT " should be in [1, 3]", dim);
3579   PetscCall(PetscOptionsBool("-dm_plex_simplex", "Mesh cell shape", "", simplex, &simplex, &flg));
3580   PetscCall(PetscOptionsBool("-dm_plex_interpolate", "Flag to create edges and faces automatically", "", interpolate, &interpolate, &flg));
3581   PetscCall(PetscOptionsBool("-dm_plex_adj_cone", "Set adjacency direction", "DMSetBasicAdjacency", adjCone, &adjCone, &flg));
3582   PetscCall(PetscOptionsBool("-dm_plex_adj_closure", "Set adjacency size", "DMSetBasicAdjacency", adjClosure, &adjClosure, &flg2));
3583   if (flg || flg2) PetscCall(DMSetBasicAdjacency(dm, adjCone, adjClosure));
3584 
3585   switch (cell) {
3586   case DM_POLYTOPE_POINT:
3587   case DM_POLYTOPE_SEGMENT:
3588   case DM_POLYTOPE_POINT_PRISM_TENSOR:
3589   case DM_POLYTOPE_TRIANGLE:
3590   case DM_POLYTOPE_QUADRILATERAL:
3591   case DM_POLYTOPE_TETRAHEDRON:
3592   case DM_POLYTOPE_HEXAHEDRON:
3593     *useCoordSpace = PETSC_TRUE;
3594     break;
3595   default:
3596     *useCoordSpace = PETSC_FALSE;
3597     break;
3598   }
3599 
3600   if (fflg) {
3601     DM dmnew;
3602 
3603     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), filename, plexname, interpolate, &dmnew));
3604     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3605     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3606   } else if (refDomain) {
3607     PetscCall(DMPlexCreateReferenceCell_Internal(dm, cell));
3608   } else if (bdfflg) {
3609     DM bdm, dmnew;
3610 
3611     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), bdFilename, plexname, interpolate, &bdm));
3612     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)bdm, "bd_"));
3613     PetscCall(DMSetFromOptions(bdm));
3614     PetscCall(DMPlexGenerate(bdm, NULL, interpolate, &dmnew));
3615     PetscCall(DMDestroy(&bdm));
3616     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3617     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3618   } else {
3619     PetscCall(PetscObjectSetName((PetscObject)dm, DMPlexShapes[shape]));
3620     switch (shape) {
3621     case DM_SHAPE_BOX:
3622     case DM_SHAPE_ANNULUS: {
3623       PetscInt       faces[3]  = {0, 0, 0};
3624       PetscReal      lower[3]  = {0, 0, 0};
3625       PetscReal      upper[3]  = {1, 1, 1};
3626       DMBoundaryType bdt[3]    = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3627       PetscBool      isAnnular = shape == DM_SHAPE_ANNULUS ? PETSC_TRUE : PETSC_FALSE;
3628       PetscInt       i, n;
3629 
3630       n = dim;
3631       for (i = 0; i < dim; ++i) faces[i] = (dim == 1 ? 1 : 4 - dim);
3632       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3633       n = 3;
3634       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3635       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3636       n = 3;
3637       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3638       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3639       n = 3;
3640       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
3641       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3642 
3643       PetscCheck(!isAnnular || dim == 2, comm, PETSC_ERR_ARG_OUTOFRANGE, "Only two dimensional annuli have been implemented");
3644       if (isAnnular)
3645         for (i = 0; i < dim - 1; ++i) bdt[i] = DM_BOUNDARY_PERIODIC;
3646 
3647       switch (cell) {
3648       case DM_POLYTOPE_TRI_PRISM_TENSOR:
3649         PetscCall(DMPlexCreateWedgeBoxMesh_Internal(dm, faces, lower, upper, bdt));
3650         if (!interpolate) {
3651           DM udm;
3652 
3653           PetscCall(DMPlexUninterpolate(dm, &udm));
3654           PetscCall(DMPlexReplace_Internal(dm, &udm));
3655         }
3656         break;
3657       default:
3658         PetscCall(DMPlexCreateBoxMesh_Internal(dm, dim, simplex, faces, lower, upper, bdt, interpolate));
3659         break;
3660       }
3661       if (isAnnular) {
3662         DM          cdm;
3663         PetscDS     cds;
3664         PetscScalar bounds[2] = {lower[0], upper[0]};
3665 
3666         // Fix coordinates for annular region
3667         PetscCall(DMSetPeriodicity(dm, NULL, NULL, NULL));
3668         PetscCall(DMSetCellCoordinatesLocal(dm, NULL));
3669         PetscCall(DMSetCellCoordinates(dm, NULL));
3670         PetscCall(DMPlexCreateCoordinateSpace(dm, 1, NULL));
3671         PetscCall(DMGetCoordinateDM(dm, &cdm));
3672         PetscCall(DMGetDS(cdm, &cds));
3673         PetscCall(PetscDSSetConstants(cds, 2, bounds));
3674         PetscCall(DMPlexRemapGeometry(dm, 0.0, boxToAnnulus));
3675       }
3676     } break;
3677     case DM_SHAPE_BOX_SURFACE: {
3678       PetscInt  faces[3] = {0, 0, 0};
3679       PetscReal lower[3] = {0, 0, 0};
3680       PetscReal upper[3] = {1, 1, 1};
3681       PetscInt  i, n;
3682 
3683       n = dim + 1;
3684       for (i = 0; i < dim + 1; ++i) faces[i] = (dim + 1 == 1 ? 1 : 4 - (dim + 1));
3685       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3686       n = 3;
3687       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3688       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3689       n = 3;
3690       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3691       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3692       PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(dm, dim + 1, faces, lower, upper, interpolate));
3693     } break;
3694     case DM_SHAPE_SPHERE: {
3695       PetscReal R = 1.0;
3696 
3697       PetscCall(PetscOptionsReal("-dm_plex_sphere_radius", "Radius of the sphere", "", R, &R, &flg));
3698       PetscCall(DMPlexCreateSphereMesh_Internal(dm, dim, simplex, R));
3699     } break;
3700     case DM_SHAPE_BALL: {
3701       PetscReal R = 1.0;
3702 
3703       PetscCall(PetscOptionsReal("-dm_plex_ball_radius", "Radius of the ball", "", R, &R, &flg));
3704       PetscCall(DMPlexCreateBallMesh_Internal(dm, dim, R));
3705     } break;
3706     case DM_SHAPE_CYLINDER: {
3707       DMBoundaryType bdt = DM_BOUNDARY_NONE;
3708       PetscInt       Nw  = 6;
3709 
3710       PetscCall(PetscOptionsEnum("-dm_plex_cylinder_bd", "Boundary type in the z direction", "", DMBoundaryTypes, (PetscEnum)bdt, (PetscEnum *)&bdt, NULL));
3711       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_wedges", "Number of wedges around the cylinder", "", Nw, &Nw, NULL));
3712       switch (cell) {
3713       case DM_POLYTOPE_TRI_PRISM_TENSOR:
3714         PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(dm, Nw, interpolate));
3715         break;
3716       default:
3717         PetscCall(DMPlexCreateHexCylinderMesh_Internal(dm, bdt));
3718         break;
3719       }
3720     } break;
3721     case DM_SHAPE_SCHWARZ_P: // fallthrough
3722     case DM_SHAPE_GYROID: {
3723       PetscInt       extent[3] = {1, 1, 1}, refine = 0, layers = 0, three;
3724       PetscReal      thickness   = 0.;
3725       DMBoundaryType periodic[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3726       DMPlexTPSType  tps_type    = shape == DM_SHAPE_SCHWARZ_P ? DMPLEX_TPS_SCHWARZ_P : DMPLEX_TPS_GYROID;
3727       PetscBool      tps_distribute;
3728       PetscCall(PetscOptionsIntArray("-dm_plex_tps_extent", "Number of replicas for each of three dimensions", NULL, extent, (three = 3, &three), NULL));
3729       PetscCall(PetscOptionsInt("-dm_plex_tps_refine", "Number of refinements", NULL, refine, &refine, NULL));
3730       PetscCall(PetscOptionsEnumArray("-dm_plex_tps_periodic", "Periodicity in each of three dimensions", NULL, DMBoundaryTypes, (PetscEnum *)periodic, (three = 3, &three), NULL));
3731       PetscCall(PetscOptionsInt("-dm_plex_tps_layers", "Number of layers in volumetric extrusion (or zero to not extrude)", NULL, layers, &layers, NULL));
3732       PetscCall(PetscOptionsReal("-dm_plex_tps_thickness", "Thickness of volumetric extrusion", NULL, thickness, &thickness, NULL));
3733       PetscCall(DMPlexDistributeGetDefault(dm, &tps_distribute));
3734       PetscCall(PetscOptionsBool("-dm_plex_tps_distribute", "Distribute the 2D mesh prior to refinement and extrusion", NULL, tps_distribute, &tps_distribute, NULL));
3735       PetscCall(DMPlexCreateTPSMesh_Internal(dm, tps_type, extent, periodic, tps_distribute, refine, layers, thickness));
3736     } break;
3737     case DM_SHAPE_DOUBLET: {
3738       DM        dmnew;
3739       PetscReal rl = 0.0;
3740 
3741       PetscCall(PetscOptionsReal("-dm_plex_doublet_refinementlimit", "Refinement limit", NULL, rl, &rl, NULL));
3742       PetscCall(DMPlexCreateDoublet(PetscObjectComm((PetscObject)dm), dim, simplex, interpolate, rl, &dmnew));
3743       PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3744       PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3745     } break;
3746     default:
3747       SETERRQ(comm, PETSC_ERR_SUP, "Domain shape %s is unsupported", DMPlexShapes[shape]);
3748     }
3749   }
3750   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3751   if (!((PetscObject)dm)->name && nameflg) PetscCall(PetscObjectSetName((PetscObject)dm, plexname));
3752   PetscFunctionReturn(0);
3753 }
3754 
3755 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
3756 {
3757   DM_Plex  *mesh = (DM_Plex *)dm->data;
3758   PetscBool flg, flg2;
3759   char      bdLabel[PETSC_MAX_PATH_LEN];
3760 
3761   PetscFunctionBegin;
3762   /* Handle viewing */
3763   PetscCall(PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL));
3764   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fem", "Debug output level all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL, 0));
3765   PetscCall(PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL));
3766   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL, 0));
3767   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_locate", "Debug output level all point location computations", "DMLocatePoints", 0, &mesh->printLocate, NULL, 0));
3768   PetscCall(DMMonitorSetFromOptions(dm, "-dm_plex_monitor_throughput", "Monitor the simulation throughput", "DMPlexMonitorThroughput", DMPlexMonitorThroughput, NULL, &flg));
3769   if (flg) PetscCall(PetscLogDefaultBegin());
3770   /* Labeling */
3771   PetscCall(PetscOptionsString("-dm_plex_boundary_label", "Label to mark the mesh boundary", "", bdLabel, bdLabel, sizeof(bdLabel), &flg));
3772   if (flg) PetscCall(DMPlexCreateBoundaryLabel_Private(dm, bdLabel));
3773   /* Point Location */
3774   PetscCall(PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL));
3775   /* Partitioning and distribution */
3776   PetscCall(PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL));
3777   /* Generation and remeshing */
3778   PetscCall(PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL));
3779   /* Projection behavior */
3780   PetscCall(PetscOptionsBoundedInt("-dm_plex_max_projection_height", "Maximum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL, 0));
3781   PetscCall(PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL));
3782   /* Checking structure */
3783   {
3784     PetscBool all = PETSC_FALSE;
3785 
3786     PetscCall(PetscOptionsBool("-dm_plex_check_all", "Perform all basic checks", "DMPlexCheck", PETSC_FALSE, &all, NULL));
3787     if (all) {
3788       PetscCall(DMPlexCheck(dm));
3789     } else {
3790       PetscCall(PetscOptionsBool("-dm_plex_check_symmetry", "Check that the adjacency information in the mesh is symmetric", "DMPlexCheckSymmetry", PETSC_FALSE, &flg, &flg2));
3791       if (flg && flg2) PetscCall(DMPlexCheckSymmetry(dm));
3792       PetscCall(PetscOptionsBool("-dm_plex_check_skeleton", "Check that each cell has the correct number of vertices (only for homogeneous simplex or tensor meshes)", "DMPlexCheckSkeleton", PETSC_FALSE, &flg, &flg2));
3793       if (flg && flg2) PetscCall(DMPlexCheckSkeleton(dm, 0));
3794       PetscCall(PetscOptionsBool("-dm_plex_check_faces", "Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type", "DMPlexCheckFaces", PETSC_FALSE, &flg, &flg2));
3795       if (flg && flg2) PetscCall(DMPlexCheckFaces(dm, 0));
3796       PetscCall(PetscOptionsBool("-dm_plex_check_geometry", "Check that cells have positive volume", "DMPlexCheckGeometry", PETSC_FALSE, &flg, &flg2));
3797       if (flg && flg2) PetscCall(DMPlexCheckGeometry(dm));
3798       PetscCall(PetscOptionsBool("-dm_plex_check_pointsf", "Check some necessary conditions for PointSF", "DMPlexCheckPointSF", PETSC_FALSE, &flg, &flg2));
3799       if (flg && flg2) PetscCall(DMPlexCheckPointSF(dm, NULL, PETSC_FALSE));
3800       PetscCall(PetscOptionsBool("-dm_plex_check_interface_cones", "Check points on inter-partition interfaces have conforming order of cone points", "DMPlexCheckInterfaceCones", PETSC_FALSE, &flg, &flg2));
3801       if (flg && flg2) PetscCall(DMPlexCheckInterfaceCones(dm));
3802     }
3803     PetscCall(PetscOptionsBool("-dm_plex_check_cell_shape", "Check cell shape", "DMPlexCheckCellShape", PETSC_FALSE, &flg, &flg2));
3804     if (flg && flg2) PetscCall(DMPlexCheckCellShape(dm, PETSC_TRUE, PETSC_DETERMINE));
3805   }
3806   {
3807     PetscReal scale = 1.0;
3808 
3809     PetscCall(PetscOptionsReal("-dm_plex_scale", "Scale factor for mesh coordinates", "DMPlexScale", scale, &scale, &flg));
3810     if (flg) {
3811       Vec coordinates, coordinatesLocal;
3812 
3813       PetscCall(DMGetCoordinates(dm, &coordinates));
3814       PetscCall(DMGetCoordinatesLocal(dm, &coordinatesLocal));
3815       PetscCall(VecScale(coordinates, scale));
3816       PetscCall(VecScale(coordinatesLocal, scale));
3817     }
3818   }
3819   PetscCall(PetscPartitionerSetFromOptions(mesh->partitioner));
3820   PetscFunctionReturn(0);
3821 }
3822 
3823 PetscErrorCode DMSetFromOptions_Overlap_Plex(DM dm, PetscOptionItems *PetscOptionsObject, PetscInt *overlap)
3824 {
3825   PetscInt  numOvLabels = 16, numOvExLabels = 16;
3826   char     *ovLabelNames[16], *ovExLabelNames[16];
3827   PetscInt  numOvValues = 16, numOvExValues = 16, l;
3828   PetscBool flg;
3829 
3830   PetscFunctionBegin;
3831   PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", *overlap, overlap, NULL, 0));
3832   PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_labels", "List of overlap label names", "DMPlexDistribute", ovLabelNames, &numOvLabels, &flg));
3833   if (!flg) numOvLabels = 0;
3834   if (numOvLabels) {
3835     ((DM_Plex *)dm->data)->numOvLabels = numOvLabels;
3836     for (l = 0; l < numOvLabels; ++l) {
3837       PetscCall(DMGetLabel(dm, ovLabelNames[l], &((DM_Plex *)dm->data)->ovLabels[l]));
3838       PetscCheck(((DM_Plex *)dm->data)->ovLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovLabelNames[l]);
3839       PetscCall(PetscFree(ovLabelNames[l]));
3840     }
3841     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_values", "List of overlap label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovValues, &numOvValues, &flg));
3842     if (!flg) numOvValues = 0;
3843     PetscCheck(numOvLabels == numOvValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvLabels, numOvValues);
3844 
3845     PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_exclude_labels", "List of overlap exclude label names", "DMPlexDistribute", ovExLabelNames, &numOvExLabels, &flg));
3846     if (!flg) numOvExLabels = 0;
3847     ((DM_Plex *)dm->data)->numOvExLabels = numOvExLabels;
3848     for (l = 0; l < numOvExLabels; ++l) {
3849       PetscCall(DMGetLabel(dm, ovExLabelNames[l], &((DM_Plex *)dm->data)->ovExLabels[l]));
3850       PetscCheck(((DM_Plex *)dm->data)->ovExLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovExLabelNames[l]);
3851       PetscCall(PetscFree(ovExLabelNames[l]));
3852     }
3853     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_exclude_values", "List of overlap exclude label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovExValues, &numOvExValues, &flg));
3854     if (!flg) numOvExValues = 0;
3855     PetscCheck(numOvExLabels == numOvExValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of exclude labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvExLabels, numOvExValues);
3856   }
3857   PetscFunctionReturn(0);
3858 }
3859 
3860 static PetscErrorCode DMSetFromOptions_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
3861 {
3862   PetscFunctionList        ordlist;
3863   char                     oname[256];
3864   PetscReal                volume    = -1.0;
3865   PetscInt                 prerefine = 0, refine = 0, r, coarsen = 0, overlap = 0, extLayers = 0, dim;
3866   PetscBool                uniformOrig, created = PETSC_FALSE, uniform = PETSC_TRUE, distribute, interpolate = PETSC_TRUE, coordSpace = PETSC_TRUE, remap = PETSC_TRUE, ghostCells = PETSC_FALSE, isHierarchy, ignoreModel = PETSC_FALSE, flg;
3867   DMPlexReorderDefaultFlag reorder;
3868 
3869   PetscFunctionBegin;
3870   PetscOptionsHeadBegin(PetscOptionsObject, "DMPlex Options");
3871   if (dm->cloneOpts) goto non_refine;
3872   /* Handle automatic creation */
3873   PetscCall(DMGetDimension(dm, &dim));
3874   if (dim < 0) {
3875     PetscCall(DMPlexCreateFromOptions_Internal(PetscOptionsObject, &coordSpace, dm));
3876     created = PETSC_TRUE;
3877   }
3878   PetscCall(DMGetDimension(dm, &dim));
3879   /* Handle interpolation before distribution */
3880   PetscCall(PetscOptionsBool("-dm_plex_interpolate_pre", "Flag to interpolate mesh before distribution", "", interpolate, &interpolate, &flg));
3881   if (flg) {
3882     DMPlexInterpolatedFlag interpolated;
3883 
3884     PetscCall(DMPlexIsInterpolated(dm, &interpolated));
3885     if (interpolated == DMPLEX_INTERPOLATED_FULL && !interpolate) {
3886       DM udm;
3887 
3888       PetscCall(DMPlexUninterpolate(dm, &udm));
3889       PetscCall(DMPlexReplace_Internal(dm, &udm));
3890     } else if (interpolated != DMPLEX_INTERPOLATED_FULL && interpolate) {
3891       DM idm;
3892 
3893       PetscCall(DMPlexInterpolate(dm, &idm));
3894       PetscCall(DMPlexReplace_Internal(dm, &idm));
3895     }
3896   }
3897   /* Handle DMPlex refinement before distribution */
3898   PetscCall(PetscOptionsBool("-dm_refine_ignore_model", "Flag to ignore the geometry model when refining", "DMCreate", ignoreModel, &ignoreModel, &flg));
3899   if (flg) ((DM_Plex *)dm->data)->ignoreModel = ignoreModel;
3900   PetscCall(DMPlexGetRefinementUniform(dm, &uniformOrig));
3901   PetscCall(PetscOptionsBoundedInt("-dm_refine_pre", "The number of refinements before distribution", "DMCreate", prerefine, &prerefine, NULL, 0));
3902   PetscCall(PetscOptionsBool("-dm_refine_remap_pre", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
3903   PetscCall(PetscOptionsBool("-dm_refine_uniform_pre", "Flag for uniform refinement before distribution", "DMCreate", uniform, &uniform, &flg));
3904   if (flg) PetscCall(DMPlexSetRefinementUniform(dm, uniform));
3905   PetscCall(PetscOptionsReal("-dm_refine_volume_limit_pre", "The maximum cell volume after refinement before distribution", "DMCreate", volume, &volume, &flg));
3906   if (flg) {
3907     PetscCall(DMPlexSetRefinementUniform(dm, PETSC_FALSE));
3908     PetscCall(DMPlexSetRefinementLimit(dm, volume));
3909     prerefine = PetscMax(prerefine, 1);
3910   }
3911   for (r = 0; r < prerefine; ++r) {
3912     DM             rdm;
3913     PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
3914 
3915     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3916     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
3917     PetscCall(DMPlexReplace_Internal(dm, &rdm));
3918     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3919     if (coordFunc && remap) {
3920       PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
3921       ((DM_Plex *)dm->data)->coordFunc = coordFunc;
3922     }
3923   }
3924   PetscCall(DMPlexSetRefinementUniform(dm, uniformOrig));
3925   /* Handle DMPlex extrusion before distribution */
3926   PetscCall(PetscOptionsBoundedInt("-dm_extrude", "The number of layers to extrude", "", extLayers, &extLayers, NULL, 0));
3927   if (extLayers) {
3928     DM edm;
3929 
3930     PetscCall(DMExtrude(dm, extLayers, &edm));
3931     PetscCall(DMPlexReplace_Internal(dm, &edm));
3932     ((DM_Plex *)dm->data)->coordFunc = NULL;
3933     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3934     extLayers = 0;
3935     PetscCall(DMGetDimension(dm, &dim));
3936   }
3937   /* Handle DMPlex reordering before distribution */
3938   PetscCall(DMPlexReorderGetDefault(dm, &reorder));
3939   PetscCall(MatGetOrderingList(&ordlist));
3940   PetscCall(PetscStrncpy(oname, MATORDERINGNATURAL, sizeof(oname)));
3941   PetscCall(PetscOptionsFList("-dm_plex_reorder", "Set mesh reordering type", "DMPlexGetOrdering", ordlist, MATORDERINGNATURAL, oname, sizeof(oname), &flg));
3942   if (reorder == DMPLEX_REORDER_DEFAULT_TRUE || flg) {
3943     DM pdm;
3944     IS perm;
3945 
3946     PetscCall(DMPlexGetOrdering(dm, oname, NULL, &perm));
3947     PetscCall(DMPlexPermute(dm, perm, &pdm));
3948     PetscCall(ISDestroy(&perm));
3949     PetscCall(DMPlexReplace_Internal(dm, &pdm));
3950     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3951   }
3952   /* Handle DMPlex distribution */
3953   PetscCall(DMPlexDistributeGetDefault(dm, &distribute));
3954   PetscCall(PetscOptionsBool("-dm_distribute", "Flag to redistribute a mesh among processes", "DMPlexDistribute", distribute, &distribute, NULL));
3955   PetscCall(DMSetFromOptions_Overlap_Plex(dm, PetscOptionsObject, &overlap));
3956   if (distribute) {
3957     DM               pdm = NULL;
3958     PetscPartitioner part;
3959 
3960     PetscCall(DMPlexGetPartitioner(dm, &part));
3961     PetscCall(PetscPartitionerSetFromOptions(part));
3962     PetscCall(DMPlexDistribute(dm, overlap, NULL, &pdm));
3963     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
3964   }
3965   /* Create coordinate space */
3966   if (created) {
3967     DM_Plex  *mesh   = (DM_Plex *)dm->data;
3968     PetscInt  degree = 1;
3969     PetscBool flg;
3970 
3971     PetscCall(PetscOptionsBool("-dm_coord_space", "Use an FEM space for coordinates", "", coordSpace, &coordSpace, &flg));
3972     PetscCall(PetscOptionsInt("-dm_coord_petscspace_degree", "FEM degree for coordinate space", "", degree, &degree, NULL));
3973     if (coordSpace) PetscCall(DMPlexCreateCoordinateSpace(dm, degree, mesh->coordFunc));
3974     if (flg && !coordSpace) {
3975       DM           cdm;
3976       PetscDS      cds;
3977       PetscObject  obj;
3978       PetscClassId id;
3979 
3980       PetscCall(DMGetCoordinateDM(dm, &cdm));
3981       PetscCall(DMGetDS(cdm, &cds));
3982       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
3983       PetscCall(PetscObjectGetClassId(obj, &id));
3984       if (id == PETSCFE_CLASSID) {
3985         PetscContainer dummy;
3986 
3987         PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &dummy));
3988         PetscCall(PetscObjectSetName((PetscObject)dummy, "coordinates"));
3989         PetscCall(DMSetField(cdm, 0, NULL, (PetscObject)dummy));
3990         PetscCall(PetscContainerDestroy(&dummy));
3991         PetscCall(DMClearDS(cdm));
3992       }
3993       mesh->coordFunc = NULL;
3994     }
3995     PetscCall(PetscOptionsBool("-dm_sparse_localize", "Localize only necessary cells", "", dm->sparseLocalize, &dm->sparseLocalize, &flg));
3996     PetscCall(DMLocalizeCoordinates(dm));
3997   }
3998   /* Handle DMPlex refinement */
3999   remap = PETSC_TRUE;
4000   PetscCall(PetscOptionsBoundedInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL, 0));
4001   PetscCall(PetscOptionsBool("-dm_refine_remap", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
4002   PetscCall(PetscOptionsBoundedInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy, 0));
4003   if (refine) PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
4004   if (refine && isHierarchy) {
4005     DM *dms, coarseDM;
4006 
4007     PetscCall(DMGetCoarseDM(dm, &coarseDM));
4008     PetscCall(PetscObjectReference((PetscObject)coarseDM));
4009     PetscCall(PetscMalloc1(refine, &dms));
4010     PetscCall(DMRefineHierarchy(dm, refine, dms));
4011     /* Total hack since we do not pass in a pointer */
4012     PetscCall(DMPlexSwap_Static(dm, dms[refine - 1]));
4013     if (refine == 1) {
4014       PetscCall(DMSetCoarseDM(dm, dms[0]));
4015       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4016     } else {
4017       PetscCall(DMSetCoarseDM(dm, dms[refine - 2]));
4018       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4019       PetscCall(DMSetCoarseDM(dms[0], dms[refine - 1]));
4020       PetscCall(DMPlexSetRegularRefinement(dms[0], PETSC_TRUE));
4021     }
4022     PetscCall(DMSetCoarseDM(dms[refine - 1], coarseDM));
4023     PetscCall(PetscObjectDereference((PetscObject)coarseDM));
4024     /* Free DMs */
4025     for (r = 0; r < refine; ++r) {
4026       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4027       PetscCall(DMDestroy(&dms[r]));
4028     }
4029     PetscCall(PetscFree(dms));
4030   } else {
4031     for (r = 0; r < refine; ++r) {
4032       DM             rdm;
4033       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4034 
4035       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4036       PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
4037       /* Total hack since we do not pass in a pointer */
4038       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4039       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4040       if (coordFunc && remap) {
4041         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4042         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4043       }
4044     }
4045   }
4046   /* Handle DMPlex coarsening */
4047   PetscCall(PetscOptionsBoundedInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL, 0));
4048   PetscCall(PetscOptionsBoundedInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy, 0));
4049   if (coarsen && isHierarchy) {
4050     DM *dms;
4051 
4052     PetscCall(PetscMalloc1(coarsen, &dms));
4053     PetscCall(DMCoarsenHierarchy(dm, coarsen, dms));
4054     /* Free DMs */
4055     for (r = 0; r < coarsen; ++r) {
4056       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4057       PetscCall(DMDestroy(&dms[r]));
4058     }
4059     PetscCall(PetscFree(dms));
4060   } else {
4061     for (r = 0; r < coarsen; ++r) {
4062       DM             cdm;
4063       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4064 
4065       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4066       PetscCall(DMCoarsen(dm, PetscObjectComm((PetscObject)dm), &cdm));
4067       /* Total hack since we do not pass in a pointer */
4068       PetscCall(DMPlexReplace_Internal(dm, &cdm));
4069       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4070       if (coordFunc) {
4071         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4072         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4073       }
4074     }
4075   }
4076   /* Handle ghost cells */
4077   PetscCall(PetscOptionsBool("-dm_plex_create_fv_ghost_cells", "Flag to create finite volume ghost cells on the boundary", "DMCreate", ghostCells, &ghostCells, NULL));
4078   if (ghostCells) {
4079     DM   gdm;
4080     char lname[PETSC_MAX_PATH_LEN];
4081 
4082     lname[0] = '\0';
4083     PetscCall(PetscOptionsString("-dm_plex_fv_ghost_cells_label", "Label name for ghost cells boundary", "DMCreate", lname, lname, sizeof(lname), &flg));
4084     PetscCall(DMPlexConstructGhostCells(dm, flg ? lname : NULL, NULL, &gdm));
4085     PetscCall(DMPlexReplace_Internal(dm, &gdm));
4086   }
4087   /* Handle 1D order */
4088   if (reorder != DMPLEX_REORDER_DEFAULT_FALSE && dim == 1) {
4089     DM           cdm, rdm;
4090     PetscDS      cds;
4091     PetscObject  obj;
4092     PetscClassId id = PETSC_OBJECT_CLASSID;
4093     IS           perm;
4094     PetscInt     Nf;
4095     PetscBool    distributed;
4096 
4097     PetscCall(DMPlexIsDistributed(dm, &distributed));
4098     PetscCall(DMGetCoordinateDM(dm, &cdm));
4099     PetscCall(DMGetDS(cdm, &cds));
4100     PetscCall(PetscDSGetNumFields(cds, &Nf));
4101     if (Nf) {
4102       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
4103       PetscCall(PetscObjectGetClassId(obj, &id));
4104     }
4105     if (!distributed && id != PETSCFE_CLASSID) {
4106       PetscCall(DMPlexGetOrdering1D(dm, &perm));
4107       PetscCall(DMPlexPermute(dm, perm, &rdm));
4108       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4109       PetscCall(ISDestroy(&perm));
4110     }
4111   }
4112 /* Handle */
4113 non_refine:
4114   PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4115   PetscOptionsHeadEnd();
4116   PetscFunctionReturn(0);
4117 }
4118 
4119 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm, Vec *vec)
4120 {
4121   PetscFunctionBegin;
4122   PetscCall(DMCreateGlobalVector_Section_Private(dm, vec));
4123   /* PetscCall(VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM)); */
4124   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex));
4125   PetscCall(VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void))VecView_Plex_Native));
4126   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex));
4127   PetscCall(VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void))VecLoad_Plex_Native));
4128   PetscFunctionReturn(0);
4129 }
4130 
4131 static PetscErrorCode DMCreateLocalVector_Plex(DM dm, Vec *vec)
4132 {
4133   PetscFunctionBegin;
4134   PetscCall(DMCreateLocalVector_Section_Private(dm, vec));
4135   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex_Local));
4136   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex_Local));
4137   PetscFunctionReturn(0);
4138 }
4139 
4140 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
4141 {
4142   PetscInt depth, d;
4143 
4144   PetscFunctionBegin;
4145   PetscCall(DMPlexGetDepth(dm, &depth));
4146   if (depth == 1) {
4147     PetscCall(DMGetDimension(dm, &d));
4148     if (dim == 0) PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4149     else if (dim == d) PetscCall(DMPlexGetDepthStratum(dm, 1, pStart, pEnd));
4150     else {
4151       *pStart = 0;
4152       *pEnd   = 0;
4153     }
4154   } else {
4155     PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4156   }
4157   PetscFunctionReturn(0);
4158 }
4159 
4160 static PetscErrorCode DMGetNeighbors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
4161 {
4162   PetscSF            sf;
4163   PetscInt           niranks, njranks, n;
4164   const PetscMPIInt *iranks, *jranks;
4165   DM_Plex           *data = (DM_Plex *)dm->data;
4166 
4167   PetscFunctionBegin;
4168   PetscCall(DMGetPointSF(dm, &sf));
4169   if (!data->neighbors) {
4170     PetscCall(PetscSFSetUp(sf));
4171     PetscCall(PetscSFGetRootRanks(sf, &njranks, &jranks, NULL, NULL, NULL));
4172     PetscCall(PetscSFGetLeafRanks(sf, &niranks, &iranks, NULL, NULL));
4173     PetscCall(PetscMalloc1(njranks + niranks + 1, &data->neighbors));
4174     PetscCall(PetscArraycpy(data->neighbors + 1, jranks, njranks));
4175     PetscCall(PetscArraycpy(data->neighbors + njranks + 1, iranks, niranks));
4176     n = njranks + niranks;
4177     PetscCall(PetscSortRemoveDupsMPIInt(&n, data->neighbors + 1));
4178     /* The following cast should never fail: can't have more neighbors than PETSC_MPI_INT_MAX */
4179     PetscCall(PetscMPIIntCast(n, data->neighbors));
4180   }
4181   if (nranks) *nranks = data->neighbors[0];
4182   if (ranks) {
4183     if (data->neighbors[0]) *ranks = data->neighbors + 1;
4184     else *ranks = NULL;
4185   }
4186   PetscFunctionReturn(0);
4187 }
4188 
4189 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM, DM, Mat, Vec, Vec);
4190 
4191 static PetscErrorCode DMInitialize_Plex(DM dm)
4192 {
4193   PetscFunctionBegin;
4194   dm->ops->view                      = DMView_Plex;
4195   dm->ops->load                      = DMLoad_Plex;
4196   dm->ops->setfromoptions            = DMSetFromOptions_Plex;
4197   dm->ops->clone                     = DMClone_Plex;
4198   dm->ops->setup                     = DMSetUp_Plex;
4199   dm->ops->createlocalsection        = DMCreateLocalSection_Plex;
4200   dm->ops->createdefaultconstraints  = DMCreateDefaultConstraints_Plex;
4201   dm->ops->createglobalvector        = DMCreateGlobalVector_Plex;
4202   dm->ops->createlocalvector         = DMCreateLocalVector_Plex;
4203   dm->ops->getlocaltoglobalmapping   = NULL;
4204   dm->ops->createfieldis             = NULL;
4205   dm->ops->createcoordinatedm        = DMCreateCoordinateDM_Plex;
4206   dm->ops->createcoordinatefield     = DMCreateCoordinateField_Plex;
4207   dm->ops->getcoloring               = NULL;
4208   dm->ops->creatematrix              = DMCreateMatrix_Plex;
4209   dm->ops->createinterpolation       = DMCreateInterpolation_Plex;
4210   dm->ops->createmassmatrix          = DMCreateMassMatrix_Plex;
4211   dm->ops->createmassmatrixlumped    = DMCreateMassMatrixLumped_Plex;
4212   dm->ops->createinjection           = DMCreateInjection_Plex;
4213   dm->ops->refine                    = DMRefine_Plex;
4214   dm->ops->coarsen                   = DMCoarsen_Plex;
4215   dm->ops->refinehierarchy           = DMRefineHierarchy_Plex;
4216   dm->ops->coarsenhierarchy          = DMCoarsenHierarchy_Plex;
4217   dm->ops->extrude                   = DMExtrude_Plex;
4218   dm->ops->globaltolocalbegin        = NULL;
4219   dm->ops->globaltolocalend          = NULL;
4220   dm->ops->localtoglobalbegin        = NULL;
4221   dm->ops->localtoglobalend          = NULL;
4222   dm->ops->destroy                   = DMDestroy_Plex;
4223   dm->ops->createsubdm               = DMCreateSubDM_Plex;
4224   dm->ops->createsuperdm             = DMCreateSuperDM_Plex;
4225   dm->ops->getdimpoints              = DMGetDimPoints_Plex;
4226   dm->ops->locatepoints              = DMLocatePoints_Plex;
4227   dm->ops->projectfunctionlocal      = DMProjectFunctionLocal_Plex;
4228   dm->ops->projectfunctionlabellocal = DMProjectFunctionLabelLocal_Plex;
4229   dm->ops->projectfieldlocal         = DMProjectFieldLocal_Plex;
4230   dm->ops->projectfieldlabellocal    = DMProjectFieldLabelLocal_Plex;
4231   dm->ops->projectbdfieldlabellocal  = DMProjectBdFieldLabelLocal_Plex;
4232   dm->ops->computel2diff             = DMComputeL2Diff_Plex;
4233   dm->ops->computel2gradientdiff     = DMComputeL2GradientDiff_Plex;
4234   dm->ops->computel2fielddiff        = DMComputeL2FieldDiff_Plex;
4235   dm->ops->getneighbors              = DMGetNeighbors_Plex;
4236   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertBoundaryValues_C", DMPlexInsertBoundaryValues_Plex));
4237   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertTimeDerviativeBoundaryValues_C", DMPlexInsertTimeDerivativeBoundaryValues_Plex));
4238   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMSetUpGLVisViewer_C", DMSetUpGLVisViewer_Plex));
4239   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMCreateNeumannOverlap_C", DMCreateNeumannOverlap_Plex));
4240   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4241   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeGetDefault_C", DMPlexDistributeGetDefault_Plex));
4242   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeSetDefault_C", DMPlexDistributeSetDefault_Plex));
4243   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderGetDefault_C", DMPlexReorderGetDefault_Plex));
4244   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderSetDefault_C", DMPlexReorderSetDefault_Plex));
4245   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMInterpolateSolution_C", DMInterpolateSolution_Plex));
4246   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4247   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetOverlap_C", DMPlexSetOverlap_Plex));
4248   PetscFunctionReturn(0);
4249 }
4250 
4251 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
4252 {
4253   DM_Plex *mesh = (DM_Plex *)dm->data;
4254 
4255   PetscFunctionBegin;
4256   mesh->refct++;
4257   (*newdm)->data = mesh;
4258   PetscCall(PetscObjectChangeTypeName((PetscObject)*newdm, DMPLEX));
4259   PetscCall(DMInitialize_Plex(*newdm));
4260   PetscFunctionReturn(0);
4261 }
4262 
4263 /*MC
4264   DMPLEX = "plex" - A DM object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
4265                     In the local representation, Vecs contain all unknowns in the interior and shared boundary. This is
4266                     specified by a PetscSection object. Ownership in the global representation is determined by
4267                     ownership of the underlying DMPlex points. This is specified by another PetscSection object.
4268 
4269   Options Database Keys:
4270 + -dm_refine_pre                     - Refine mesh before distribution
4271 + -dm_refine_uniform_pre             - Choose uniform or generator-based refinement
4272 + -dm_refine_volume_limit_pre        - Cell volume limit after pre-refinement using generator
4273 . -dm_distribute                     - Distribute mesh across processes
4274 . -dm_distribute_overlap             - Number of cells to overlap for distribution
4275 . -dm_refine                         - Refine mesh after distribution
4276 . -dm_plex_hash_location             - Use grid hashing for point location
4277 . -dm_plex_hash_box_faces <n,m,p>    - The number of divisions in each direction of the grid hash
4278 . -dm_plex_partition_balance         - Attempt to evenly divide points on partition boundary between processes
4279 . -dm_plex_remesh_bd                 - Allow changes to the boundary on remeshing
4280 . -dm_plex_max_projection_height     - Maximum mesh point height used to project locally
4281 . -dm_plex_regular_refinement        - Use special nested projection algorithm for regular refinement
4282 . -dm_plex_check_all                 - Perform all shecks below
4283 . -dm_plex_check_symmetry            - Check that the adjacency information in the mesh is symmetric
4284 . -dm_plex_check_skeleton <celltype> - Check that each cell has the correct number of vertices
4285 . -dm_plex_check_faces <celltype>    - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
4286 . -dm_plex_check_geometry            - Check that cells have positive volume
4287 . -dm_view :mesh.tex:ascii_latex     - View the mesh in LaTeX/TikZ
4288 . -dm_plex_view_scale <num>          - Scale the TikZ
4289 - -dm_plex_print_fem <num>           - View FEM assembly information, such as element vectors and matrices
4290 
4291   Level: intermediate
4292 
4293 .seealso: `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`
4294 M*/
4295 
4296 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
4297 {
4298   DM_Plex *mesh;
4299   PetscInt unit;
4300 
4301   PetscFunctionBegin;
4302   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4303   PetscCall(PetscNew(&mesh));
4304   dm->data = mesh;
4305 
4306   mesh->refct = 1;
4307   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection));
4308   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection));
4309   mesh->refinementUniform      = PETSC_TRUE;
4310   mesh->refinementLimit        = -1.0;
4311   mesh->distDefault            = PETSC_TRUE;
4312   mesh->reorderDefault         = DMPLEX_REORDER_DEFAULT_NOTSET;
4313   mesh->distributionName       = NULL;
4314   mesh->interpolated           = DMPLEX_INTERPOLATED_INVALID;
4315   mesh->interpolatedCollective = DMPLEX_INTERPOLATED_INVALID;
4316 
4317   PetscCall(PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner));
4318   mesh->remeshBd = PETSC_FALSE;
4319 
4320   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
4321 
4322   mesh->depthState    = -1;
4323   mesh->celltypeState = -1;
4324   mesh->printTol      = 1.0e-10;
4325 
4326   PetscCall(DMInitialize_Plex(dm));
4327   PetscFunctionReturn(0);
4328 }
4329 
4330 /*@
4331   DMPlexCreate - Creates a DMPlex object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
4332 
4333   Collective
4334 
4335   Input Parameter:
4336 . comm - The communicator for the DMPlex object
4337 
4338   Output Parameter:
4339 . mesh  - The DMPlex object
4340 
4341   Level: beginner
4342 
4343 @*/
4344 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
4345 {
4346   PetscFunctionBegin;
4347   PetscValidPointer(mesh, 2);
4348   PetscCall(DMCreate(comm, mesh));
4349   PetscCall(DMSetType(*mesh, DMPLEX));
4350   PetscFunctionReturn(0);
4351 }
4352 
4353 /*@C
4354   DMPlexBuildFromCellListParallel - Build distributed DMPLEX topology from a list of vertices for each cell (common mesh generator output)
4355 
4356   Input Parameters:
4357 + dm - The DM
4358 . numCells - The number of cells owned by this process
4359 . numVertices - The number of vertices to be owned by this process, or PETSC_DECIDE
4360 . NVertices - The global number of vertices, or PETSC_DETERMINE
4361 . numCorners - The number of vertices for each cell
4362 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4363 
4364   Output Parameters:
4365 + vertexSF - (Optional) SF describing complete vertex ownership
4366 - verticesAdjSaved - (Optional) vertex adjacency array
4367 
4368   Notes:
4369   Two triangles sharing a face
4370 $
4371 $        2
4372 $      / | \
4373 $     /  |  \
4374 $    /   |   \
4375 $   0  0 | 1  3
4376 $    \   |   /
4377 $     \  |  /
4378 $      \ | /
4379 $        1
4380 would have input
4381 $  numCells = 2, numVertices = 4
4382 $  cells = [0 1 2  1 3 2]
4383 $
4384 which would result in the DMPlex
4385 $
4386 $        4
4387 $      / | \
4388 $     /  |  \
4389 $    /   |   \
4390 $   2  0 | 1  5
4391 $    \   |   /
4392 $     \  |  /
4393 $      \ | /
4394 $        3
4395 
4396   Vertices are implicitly numbered consecutively 0,...,NVertices.
4397   Each rank owns a chunk of numVertices consecutive vertices.
4398   If numVertices is PETSC_DECIDE, PETSc will distribute them as evenly as possible using PetscLayout.
4399   If NVertices is PETSC_DETERMINE and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
4400   If only NVertices is PETSC_DETERMINE, it is computed as the sum of numVertices over all ranks.
4401 
4402   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
4403 
4404   Not currently supported in Fortran.
4405 
4406   Level: advanced
4407 
4408 .seealso: `DMPlexBuildFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildCoordinatesFromCellListParallel()`
4409 @*/
4410 PetscErrorCode DMPlexBuildFromCellListParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, const PetscInt cells[], PetscSF *vertexSF, PetscInt **verticesAdjSaved)
4411 {
4412   PetscSF     sfPoint;
4413   PetscLayout layout;
4414   PetscInt    numVerticesAdj, *verticesAdj, *cones, c, p;
4415 
4416   PetscFunctionBegin;
4417   PetscValidLogicalCollectiveInt(dm, NVertices, 4);
4418   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4419   /* Get/check global number of vertices */
4420   {
4421     PetscInt       NVerticesInCells, i;
4422     const PetscInt len = numCells * numCorners;
4423 
4424     /* NVerticesInCells = max(cells) + 1 */
4425     NVerticesInCells = PETSC_MIN_INT;
4426     for (i = 0; i < len; i++)
4427       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4428     ++NVerticesInCells;
4429     PetscCallMPI(MPI_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
4430 
4431     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
4432     else
4433       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
4434   }
4435   /* Count locally unique vertices */
4436   {
4437     PetscHSetI vhash;
4438     PetscInt   off = 0;
4439 
4440     PetscCall(PetscHSetICreate(&vhash));
4441     for (c = 0; c < numCells; ++c) {
4442       for (p = 0; p < numCorners; ++p) PetscCall(PetscHSetIAdd(vhash, cells[c * numCorners + p]));
4443     }
4444     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
4445     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
4446     else verticesAdj = *verticesAdjSaved;
4447     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
4448     PetscCall(PetscHSetIDestroy(&vhash));
4449     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
4450   }
4451   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
4452   /* Create cones */
4453   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
4454   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
4455   PetscCall(DMSetUp(dm));
4456   PetscCall(DMPlexGetCones(dm, &cones));
4457   for (c = 0; c < numCells; ++c) {
4458     for (p = 0; p < numCorners; ++p) {
4459       const PetscInt gv = cells[c * numCorners + p];
4460       PetscInt       lv;
4461 
4462       /* Positions within verticesAdj form 0-based local vertex numbering;
4463          we need to shift it by numCells to get correct DAG points (cells go first) */
4464       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
4465       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
4466       cones[c * numCorners + p] = lv + numCells;
4467     }
4468   }
4469   /* Build point sf */
4470   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
4471   PetscCall(PetscLayoutSetSize(layout, NVertices));
4472   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
4473   PetscCall(PetscLayoutSetBlockSize(layout, 1));
4474   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
4475   PetscCall(PetscLayoutDestroy(&layout));
4476   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
4477   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
4478   if (dm->sf) {
4479     const char *prefix;
4480 
4481     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
4482     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
4483   }
4484   PetscCall(DMSetPointSF(dm, sfPoint));
4485   PetscCall(PetscSFDestroy(&sfPoint));
4486   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)(*vertexSF), "Vertex Ownership SF"));
4487   /* Fill in the rest of the topology structure */
4488   PetscCall(DMPlexSymmetrize(dm));
4489   PetscCall(DMPlexStratify(dm));
4490   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4491   PetscFunctionReturn(0);
4492 }
4493 
4494 /*@C
4495   DMPlexBuildCoordinatesFromCellListParallel - Build DM coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4496 
4497   Input Parameters:
4498 + dm - The DM
4499 . spaceDim - The spatial dimension used for coordinates
4500 . sfVert - SF describing complete vertex ownership
4501 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4502 
4503   Level: advanced
4504 
4505   Notes:
4506   Not currently supported in Fortran.
4507 
4508 .seealso: `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellListParallel()`
4509 @*/
4510 PetscErrorCode DMPlexBuildCoordinatesFromCellListParallel(DM dm, PetscInt spaceDim, PetscSF sfVert, const PetscReal vertexCoords[])
4511 {
4512   PetscSection coordSection;
4513   Vec          coordinates;
4514   PetscScalar *coords;
4515   PetscInt     numVertices, numVerticesAdj, coordSize, v, vStart, vEnd;
4516 
4517   PetscFunctionBegin;
4518   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4519   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
4520   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4521   PetscCall(DMSetCoordinateDim(dm, spaceDim));
4522   PetscCall(PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL));
4523   PetscCheck(vEnd - vStart == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Supplied sfVert has wrong number of leaves = %" PetscInt_FMT " != %" PetscInt_FMT " = vEnd - vStart", numVerticesAdj, vEnd - vStart);
4524   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4525   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4526   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
4527   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
4528   for (v = vStart; v < vEnd; ++v) {
4529     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
4530     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
4531   }
4532   PetscCall(PetscSectionSetUp(coordSection));
4533   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4534   PetscCall(VecCreate(PetscObjectComm((PetscObject)dm), &coordinates));
4535   PetscCall(VecSetBlockSize(coordinates, spaceDim));
4536   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4537   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4538   PetscCall(VecSetType(coordinates, VECSTANDARD));
4539   PetscCall(VecGetArray(coordinates, &coords));
4540   {
4541     MPI_Datatype coordtype;
4542 
4543     /* Need a temp buffer for coords if we have complex/single */
4544     PetscCallMPI(MPI_Type_contiguous(spaceDim, MPIU_SCALAR, &coordtype));
4545     PetscCallMPI(MPI_Type_commit(&coordtype));
4546 #if defined(PETSC_USE_COMPLEX)
4547     {
4548       PetscScalar *svertexCoords;
4549       PetscInt     i;
4550       PetscCall(PetscMalloc1(numVertices * spaceDim, &svertexCoords));
4551       for (i = 0; i < numVertices * spaceDim; i++) svertexCoords[i] = vertexCoords[i];
4552       PetscCall(PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4553       PetscCall(PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4554       PetscCall(PetscFree(svertexCoords));
4555     }
4556 #else
4557     PetscCall(PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4558     PetscCall(PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4559 #endif
4560     PetscCallMPI(MPI_Type_free(&coordtype));
4561   }
4562   PetscCall(VecRestoreArray(coordinates, &coords));
4563   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4564   PetscCall(VecDestroy(&coordinates));
4565   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4566   PetscFunctionReturn(0);
4567 }
4568 
4569 /*@
4570   DMPlexCreateFromCellListParallelPetsc - Create distributed DMPLEX from a list of vertices for each cell (common mesh generator output)
4571 
4572   Input Parameters:
4573 + comm - The communicator
4574 . dim - The topological dimension of the mesh
4575 . numCells - The number of cells owned by this process
4576 . numVertices - The number of vertices owned by this process, or PETSC_DECIDE
4577 . NVertices - The global number of vertices, or PETSC_DECIDE
4578 . numCorners - The number of vertices for each cell
4579 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4580 . cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4581 . spaceDim - The spatial dimension used for coordinates
4582 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4583 
4584   Output Parameters:
4585 + dm - The DM
4586 . vertexSF - (Optional) SF describing complete vertex ownership
4587 - verticesAdjSaved - (Optional) vertex adjacency array
4588 
4589   Notes:
4590   This function is just a convenient sequence of DMCreate(), DMSetType(), DMSetDimension(),
4591   DMPlexBuildFromCellListParallel(), DMPlexInterpolate(), DMPlexBuildCoordinatesFromCellListParallel()
4592 
4593   See DMPlexBuildFromCellListParallel() for an example and details about the topology-related parameters.
4594   See DMPlexBuildCoordinatesFromCellListParallel() for details about the geometry-related parameters.
4595 
4596   Level: intermediate
4597 
4598 .seealso: `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
4599 @*/
4600 PetscErrorCode DMPlexCreateFromCellListParallelPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, PetscInt **verticesAdj, DM *dm)
4601 {
4602   PetscSF sfVert;
4603 
4604   PetscFunctionBegin;
4605   PetscCall(DMCreate(comm, dm));
4606   PetscCall(DMSetType(*dm, DMPLEX));
4607   PetscValidLogicalCollectiveInt(*dm, dim, 2);
4608   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
4609   PetscCall(DMSetDimension(*dm, dim));
4610   PetscCall(DMPlexBuildFromCellListParallel(*dm, numCells, numVertices, NVertices, numCorners, cells, &sfVert, verticesAdj));
4611   if (interpolate) {
4612     DM idm;
4613 
4614     PetscCall(DMPlexInterpolate(*dm, &idm));
4615     PetscCall(DMDestroy(dm));
4616     *dm = idm;
4617   }
4618   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
4619   if (vertexSF) *vertexSF = sfVert;
4620   else PetscCall(PetscSFDestroy(&sfVert));
4621   PetscFunctionReturn(0);
4622 }
4623 
4624 /*@C
4625   DMPlexBuildFromCellList - Build DMPLEX topology from a list of vertices for each cell (common mesh generator output)
4626 
4627   Input Parameters:
4628 + dm - The DM
4629 . numCells - The number of cells owned by this process
4630 . numVertices - The number of vertices owned by this process, or PETSC_DETERMINE
4631 . numCorners - The number of vertices for each cell
4632 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4633 
4634   Level: advanced
4635 
4636   Notes:
4637   Two triangles sharing a face
4638 $
4639 $        2
4640 $      / | \
4641 $     /  |  \
4642 $    /   |   \
4643 $   0  0 | 1  3
4644 $    \   |   /
4645 $     \  |  /
4646 $      \ | /
4647 $        1
4648 would have input
4649 $  numCells = 2, numVertices = 4
4650 $  cells = [0 1 2  1 3 2]
4651 $
4652 which would result in the DMPlex
4653 $
4654 $        4
4655 $      / | \
4656 $     /  |  \
4657 $    /   |   \
4658 $   2  0 | 1  5
4659 $    \   |   /
4660 $     \  |  /
4661 $      \ | /
4662 $        3
4663 
4664   If numVertices is PETSC_DETERMINE, it is computed by PETSc as the maximum vertex index in cells + 1.
4665 
4666   Not currently supported in Fortran.
4667 
4668 .seealso: `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListPetsc()`
4669 @*/
4670 PetscErrorCode DMPlexBuildFromCellList(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const PetscInt cells[])
4671 {
4672   PetscInt *cones, c, p, dim;
4673 
4674   PetscFunctionBegin;
4675   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4676   PetscCall(DMGetDimension(dm, &dim));
4677   /* Get/check global number of vertices */
4678   {
4679     PetscInt       NVerticesInCells, i;
4680     const PetscInt len = numCells * numCorners;
4681 
4682     /* NVerticesInCells = max(cells) + 1 */
4683     NVerticesInCells = PETSC_MIN_INT;
4684     for (i = 0; i < len; i++)
4685       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4686     ++NVerticesInCells;
4687 
4688     if (numVertices == PETSC_DECIDE) numVertices = NVerticesInCells;
4689     else
4690       PetscCheck(numVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, numVertices, NVerticesInCells);
4691   }
4692   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
4693   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
4694   PetscCall(DMSetUp(dm));
4695   PetscCall(DMPlexGetCones(dm, &cones));
4696   for (c = 0; c < numCells; ++c) {
4697     for (p = 0; p < numCorners; ++p) cones[c * numCorners + p] = cells[c * numCorners + p] + numCells;
4698   }
4699   PetscCall(DMPlexSymmetrize(dm));
4700   PetscCall(DMPlexStratify(dm));
4701   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4702   PetscFunctionReturn(0);
4703 }
4704 
4705 /*@C
4706   DMPlexBuildCoordinatesFromCellList - Build DM coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4707 
4708   Input Parameters:
4709 + dm - The DM
4710 . spaceDim - The spatial dimension used for coordinates
4711 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4712 
4713   Level: advanced
4714 
4715   Notes:
4716   Not currently supported in Fortran.
4717 
4718 .seealso: `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellList()`
4719 @*/
4720 PetscErrorCode DMPlexBuildCoordinatesFromCellList(DM dm, PetscInt spaceDim, const PetscReal vertexCoords[])
4721 {
4722   PetscSection coordSection;
4723   Vec          coordinates;
4724   DM           cdm;
4725   PetscScalar *coords;
4726   PetscInt     v, vStart, vEnd, d;
4727 
4728   PetscFunctionBegin;
4729   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4730   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
4731   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4732   PetscCall(DMSetCoordinateDim(dm, spaceDim));
4733   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4734   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4735   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
4736   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
4737   for (v = vStart; v < vEnd; ++v) {
4738     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
4739     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
4740   }
4741   PetscCall(PetscSectionSetUp(coordSection));
4742 
4743   PetscCall(DMGetCoordinateDM(dm, &cdm));
4744   PetscCall(DMCreateLocalVector(cdm, &coordinates));
4745   PetscCall(VecSetBlockSize(coordinates, spaceDim));
4746   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4747   PetscCall(VecGetArrayWrite(coordinates, &coords));
4748   for (v = 0; v < vEnd - vStart; ++v) {
4749     for (d = 0; d < spaceDim; ++d) coords[v * spaceDim + d] = vertexCoords[v * spaceDim + d];
4750   }
4751   PetscCall(VecRestoreArrayWrite(coordinates, &coords));
4752   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4753   PetscCall(VecDestroy(&coordinates));
4754   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4755   PetscFunctionReturn(0);
4756 }
4757 
4758 /*@
4759   DMPlexCreateFromCellListPetsc - Create DMPLEX from a list of vertices for each cell (common mesh generator output), but only process 0 takes in the input
4760 
4761   Collective on comm
4762 
4763   Input Parameters:
4764 + comm - The communicator
4765 . dim - The topological dimension of the mesh
4766 . numCells - The number of cells, only on process 0
4767 . numVertices - The number of vertices owned by this process, or PETSC_DECIDE, only on process 0
4768 . numCorners - The number of vertices for each cell, only on process 0
4769 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4770 . cells - An array of numCells*numCorners numbers, the vertices for each cell, only on process 0
4771 . spaceDim - The spatial dimension used for coordinates
4772 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex, only on process 0
4773 
4774   Output Parameter:
4775 . dm - The DM, which only has points on process 0
4776 
4777   Notes:
4778   This function is just a convenient sequence of DMCreate(), DMSetType(), DMSetDimension(), DMPlexBuildFromCellList(),
4779   DMPlexInterpolate(), DMPlexBuildCoordinatesFromCellList()
4780 
4781   See DMPlexBuildFromCellList() for an example and details about the topology-related parameters.
4782   See DMPlexBuildCoordinatesFromCellList() for details about the geometry-related parameters.
4783   See DMPlexCreateFromCellListParallelPetsc() for parallel input
4784 
4785   Level: intermediate
4786 
4787 .seealso: `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellList()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
4788 @*/
4789 PetscErrorCode DMPlexCreateFromCellListPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], DM *dm)
4790 {
4791   PetscMPIInt rank;
4792 
4793   PetscFunctionBegin;
4794   PetscCheck(dim, comm, PETSC_ERR_ARG_OUTOFRANGE, "This is not appropriate for 0-dimensional meshes. Consider either creating the DM using DMPlexCreateFromDAG(), by hand, or using DMSwarm.");
4795   PetscCallMPI(MPI_Comm_rank(comm, &rank));
4796   PetscCall(DMCreate(comm, dm));
4797   PetscCall(DMSetType(*dm, DMPLEX));
4798   PetscCall(DMSetDimension(*dm, dim));
4799   if (rank == 0) PetscCall(DMPlexBuildFromCellList(*dm, numCells, numVertices, numCorners, cells));
4800   else PetscCall(DMPlexBuildFromCellList(*dm, 0, 0, 0, NULL));
4801   if (interpolate) {
4802     DM idm;
4803 
4804     PetscCall(DMPlexInterpolate(*dm, &idm));
4805     PetscCall(DMDestroy(dm));
4806     *dm = idm;
4807   }
4808   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, vertexCoords));
4809   else PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, NULL));
4810   PetscFunctionReturn(0);
4811 }
4812 
4813 /*@
4814   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a DM
4815 
4816   Input Parameters:
4817 + dm - The empty DM object, usually from DMCreate() and DMSetDimension()
4818 . depth - The depth of the DAG
4819 . numPoints - Array of size depth + 1 containing the number of points at each depth
4820 . coneSize - The cone size of each point
4821 . cones - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
4822 . coneOrientations - The orientation of each cone point
4823 - vertexCoords - An array of numPoints[0]*spacedim numbers representing the coordinates of each vertex, with spacedim the value set via DMSetCoordinateDim()
4824 
4825   Output Parameter:
4826 . dm - The DM
4827 
4828   Note: Two triangles sharing a face would have input
4829 $  depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
4830 $  cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
4831 $ vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
4832 $
4833 which would result in the DMPlex
4834 $
4835 $        4
4836 $      / | \
4837 $     /  |  \
4838 $    /   |   \
4839 $   2  0 | 1  5
4840 $    \   |   /
4841 $     \  |  /
4842 $      \ | /
4843 $        3
4844 $
4845 $ Notice that all points are numbered consecutively, unlike DMPlexCreateFromCellListPetsc()
4846 
4847   Level: advanced
4848 
4849 .seealso: `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
4850 @*/
4851 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
4852 {
4853   Vec          coordinates;
4854   PetscSection coordSection;
4855   PetscScalar *coords;
4856   PetscInt     coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
4857 
4858   PetscFunctionBegin;
4859   PetscCall(DMGetDimension(dm, &dim));
4860   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
4861   PetscCheck(dimEmbed >= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Embedding dimension %" PetscInt_FMT " cannot be less than intrinsic dimension %" PetscInt_FMT, dimEmbed, dim);
4862   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
4863   PetscCall(DMPlexSetChart(dm, pStart, pEnd));
4864   for (p = pStart; p < pEnd; ++p) {
4865     PetscCall(DMPlexSetConeSize(dm, p, coneSize[p - pStart]));
4866     if (firstVertex < 0 && !coneSize[p - pStart]) firstVertex = p - pStart;
4867   }
4868   PetscCheck(firstVertex >= 0 || !numPoints[0], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Expected %" PetscInt_FMT " vertices but could not find any", numPoints[0]);
4869   PetscCall(DMSetUp(dm)); /* Allocate space for cones */
4870   for (p = pStart, off = 0; p < pEnd; off += coneSize[p - pStart], ++p) {
4871     PetscCall(DMPlexSetCone(dm, p, &cones[off]));
4872     PetscCall(DMPlexSetConeOrientation(dm, p, &coneOrientations[off]));
4873   }
4874   PetscCall(DMPlexSymmetrize(dm));
4875   PetscCall(DMPlexStratify(dm));
4876   /* Build coordinates */
4877   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4878   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4879   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dimEmbed));
4880   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numPoints[0]));
4881   for (v = firstVertex; v < firstVertex + numPoints[0]; ++v) {
4882     PetscCall(PetscSectionSetDof(coordSection, v, dimEmbed));
4883     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed));
4884   }
4885   PetscCall(PetscSectionSetUp(coordSection));
4886   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4887   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
4888   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4889   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4890   PetscCall(VecSetBlockSize(coordinates, dimEmbed));
4891   PetscCall(VecSetType(coordinates, VECSTANDARD));
4892   if (vertexCoords) {
4893     PetscCall(VecGetArray(coordinates, &coords));
4894     for (v = 0; v < numPoints[0]; ++v) {
4895       PetscInt off;
4896 
4897       PetscCall(PetscSectionGetOffset(coordSection, v + firstVertex, &off));
4898       for (d = 0; d < dimEmbed; ++d) coords[off + d] = vertexCoords[v * dimEmbed + d];
4899     }
4900   }
4901   PetscCall(VecRestoreArray(coordinates, &coords));
4902   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4903   PetscCall(VecDestroy(&coordinates));
4904   PetscFunctionReturn(0);
4905 }
4906 
4907 /*@C
4908   DMPlexCreateCellVertexFromFile - Create a DMPlex mesh from a simple cell-vertex file.
4909 
4910 + comm        - The MPI communicator
4911 . filename    - Name of the .dat file
4912 - interpolate - Create faces and edges in the mesh
4913 
4914   Output Parameter:
4915 . dm  - The DM object representing the mesh
4916 
4917   Note: The format is the simplest possible:
4918 $ Ne
4919 $ v0 v1 ... vk
4920 $ Nv
4921 $ x y z marker
4922 
4923   Level: beginner
4924 
4925 .seealso: `DMPlexCreateFromFile()`, `DMPlexCreateMedFromFile()`, `DMPlexCreateGmsh()`, `DMPlexCreate()`
4926 @*/
4927 PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
4928 {
4929   DMLabel      marker;
4930   PetscViewer  viewer;
4931   Vec          coordinates;
4932   PetscSection coordSection;
4933   PetscScalar *coords;
4934   char         line[PETSC_MAX_PATH_LEN];
4935   PetscInt     dim = 3, cdim = 3, coordSize, v, c, d;
4936   PetscMPIInt  rank;
4937   int          snum, Nv, Nc, Ncn, Nl;
4938 
4939   PetscFunctionBegin;
4940   PetscCallMPI(MPI_Comm_rank(comm, &rank));
4941   PetscCall(PetscViewerCreate(comm, &viewer));
4942   PetscCall(PetscViewerSetType(viewer, PETSCVIEWERASCII));
4943   PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
4944   PetscCall(PetscViewerFileSetName(viewer, filename));
4945   if (rank == 0) {
4946     PetscCall(PetscViewerRead(viewer, line, 4, NULL, PETSC_STRING));
4947     snum = sscanf(line, "%d %d %d %d", &Nc, &Nv, &Ncn, &Nl);
4948     PetscCheck(snum == 4, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4949   } else {
4950     Nc = Nv = Ncn = Nl = 0;
4951   }
4952   PetscCall(DMCreate(comm, dm));
4953   PetscCall(DMSetType(*dm, DMPLEX));
4954   PetscCall(DMPlexSetChart(*dm, 0, Nc + Nv));
4955   PetscCall(DMSetDimension(*dm, dim));
4956   PetscCall(DMSetCoordinateDim(*dm, cdim));
4957   /* Read topology */
4958   if (rank == 0) {
4959     char     format[PETSC_MAX_PATH_LEN];
4960     PetscInt cone[8];
4961     int      vbuf[8], v;
4962 
4963     for (c = 0; c < Ncn; ++c) {
4964       format[c * 3 + 0] = '%';
4965       format[c * 3 + 1] = 'd';
4966       format[c * 3 + 2] = ' ';
4967     }
4968     format[Ncn * 3 - 1] = '\0';
4969     for (c = 0; c < Nc; ++c) PetscCall(DMPlexSetConeSize(*dm, c, Ncn));
4970     PetscCall(DMSetUp(*dm));
4971     for (c = 0; c < Nc; ++c) {
4972       PetscCall(PetscViewerRead(viewer, line, Ncn, NULL, PETSC_STRING));
4973       switch (Ncn) {
4974       case 2:
4975         snum = sscanf(line, format, &vbuf[0], &vbuf[1]);
4976         break;
4977       case 3:
4978         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2]);
4979         break;
4980       case 4:
4981         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3]);
4982         break;
4983       case 6:
4984         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5]);
4985         break;
4986       case 8:
4987         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);
4988         break;
4989       default:
4990         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No cell shape with %d vertices", Ncn);
4991       }
4992       PetscCheck(snum == Ncn, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4993       for (v = 0; v < Ncn; ++v) cone[v] = vbuf[v] + Nc;
4994       /* Hexahedra are inverted */
4995       if (Ncn == 8) {
4996         PetscInt tmp = cone[1];
4997         cone[1]      = cone[3];
4998         cone[3]      = tmp;
4999       }
5000       PetscCall(DMPlexSetCone(*dm, c, cone));
5001     }
5002   }
5003   PetscCall(DMPlexSymmetrize(*dm));
5004   PetscCall(DMPlexStratify(*dm));
5005   /* Read coordinates */
5006   PetscCall(DMGetCoordinateSection(*dm, &coordSection));
5007   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5008   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
5009   PetscCall(PetscSectionSetChart(coordSection, Nc, Nc + Nv));
5010   for (v = Nc; v < Nc + Nv; ++v) {
5011     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
5012     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
5013   }
5014   PetscCall(PetscSectionSetUp(coordSection));
5015   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
5016   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
5017   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5018   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
5019   PetscCall(VecSetBlockSize(coordinates, cdim));
5020   PetscCall(VecSetType(coordinates, VECSTANDARD));
5021   PetscCall(VecGetArray(coordinates, &coords));
5022   if (rank == 0) {
5023     char   format[PETSC_MAX_PATH_LEN];
5024     double x[3];
5025     int    l, val[3];
5026 
5027     if (Nl) {
5028       for (l = 0; l < Nl; ++l) {
5029         format[l * 3 + 0] = '%';
5030         format[l * 3 + 1] = 'd';
5031         format[l * 3 + 2] = ' ';
5032       }
5033       format[Nl * 3 - 1] = '\0';
5034       PetscCall(DMCreateLabel(*dm, "marker"));
5035       PetscCall(DMGetLabel(*dm, "marker", &marker));
5036     }
5037     for (v = 0; v < Nv; ++v) {
5038       PetscCall(PetscViewerRead(viewer, line, 3 + Nl, NULL, PETSC_STRING));
5039       snum = sscanf(line, "%lg %lg %lg", &x[0], &x[1], &x[2]);
5040       PetscCheck(snum == 3, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5041       switch (Nl) {
5042       case 0:
5043         snum = 0;
5044         break;
5045       case 1:
5046         snum = sscanf(line, format, &val[0]);
5047         break;
5048       case 2:
5049         snum = sscanf(line, format, &val[0], &val[1]);
5050         break;
5051       case 3:
5052         snum = sscanf(line, format, &val[0], &val[1], &val[2]);
5053         break;
5054       default:
5055         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Request support for %d labels", Nl);
5056       }
5057       PetscCheck(snum == Nl, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5058       for (d = 0; d < cdim; ++d) coords[v * cdim + d] = x[d];
5059       for (l = 0; l < Nl; ++l) PetscCall(DMLabelSetValue(marker, v + Nc, val[l]));
5060     }
5061   }
5062   PetscCall(VecRestoreArray(coordinates, &coords));
5063   PetscCall(DMSetCoordinatesLocal(*dm, coordinates));
5064   PetscCall(VecDestroy(&coordinates));
5065   PetscCall(PetscViewerDestroy(&viewer));
5066   if (interpolate) {
5067     DM      idm;
5068     DMLabel bdlabel;
5069 
5070     PetscCall(DMPlexInterpolate(*dm, &idm));
5071     PetscCall(DMDestroy(dm));
5072     *dm = idm;
5073 
5074     if (!Nl) {
5075       PetscCall(DMCreateLabel(*dm, "marker"));
5076       PetscCall(DMGetLabel(*dm, "marker", &bdlabel));
5077       PetscCall(DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel));
5078       PetscCall(DMPlexLabelComplete(*dm, bdlabel));
5079     }
5080   }
5081   PetscFunctionReturn(0);
5082 }
5083 
5084 /*@C
5085   DMPlexCreateFromFile - This takes a filename and produces a DM
5086 
5087   Input Parameters:
5088 + comm - The communicator
5089 . filename - A file name
5090 . plexname - The object name of the resulting DM, also used for intra-datafile lookup by some formats
5091 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
5092 
5093   Output Parameter:
5094 . dm - The DM
5095 
5096   Options Database Keys:
5097 . -dm_plex_create_from_hdf5_xdmf - use the PETSC_VIEWER_HDF5_XDMF format for reading HDF5
5098 
5099   Use -dm_plex_create_ prefix to pass options to the internal PetscViewer, e.g.
5100 $ -dm_plex_create_viewer_hdf5_collective
5101 
5102   Notes:
5103   Using PETSCVIEWERHDF5 type with PETSC_VIEWER_HDF5_PETSC format, one can save multiple DMPlex
5104   meshes in a single HDF5 file. This in turn requires one to name the DMPlex object with PetscObjectSetName()
5105   before saving it with DMView() and before loading it with DMLoad() for identification of the mesh object.
5106   The input parameter name is thus used to name the DMPlex object when DMPlexCreateFromFile() internally
5107   calls DMLoad(). Currently, name is ignored for other viewer types and/or formats.
5108 
5109   Level: beginner
5110 
5111 .seealso: `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`, `PetscObjectSetName()`, `DMView()`, `DMLoad()`
5112 @*/
5113 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], const char plexname[], PetscBool interpolate, DM *dm)
5114 {
5115   const char  extGmsh[]      = ".msh";
5116   const char  extGmsh2[]     = ".msh2";
5117   const char  extGmsh4[]     = ".msh4";
5118   const char  extCGNS[]      = ".cgns";
5119   const char  extExodus[]    = ".exo";
5120   const char  extExodus_e[]  = ".e";
5121   const char  extGenesis[]   = ".gen";
5122   const char  extFluent[]    = ".cas";
5123   const char  extHDF5[]      = ".h5";
5124   const char  extMed[]       = ".med";
5125   const char  extPLY[]       = ".ply";
5126   const char  extEGADSLite[] = ".egadslite";
5127   const char  extEGADS[]     = ".egads";
5128   const char  extIGES[]      = ".igs";
5129   const char  extSTEP[]      = ".stp";
5130   const char  extCV[]        = ".dat";
5131   size_t      len;
5132   PetscBool   isGmsh, isGmsh2, isGmsh4, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isMed, isPLY, isEGADSLite, isEGADS, isIGES, isSTEP, isCV;
5133   PetscMPIInt rank;
5134 
5135   PetscFunctionBegin;
5136   PetscValidCharPointer(filename, 2);
5137   if (plexname) PetscValidCharPointer(plexname, 3);
5138   PetscValidPointer(dm, 5);
5139   PetscCall(DMInitializePackage());
5140   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5141   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5142   PetscCall(PetscStrlen(filename, &len));
5143   PetscCheck(len, comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
5144 
5145 #define CheckExtension(extension__, is_extension__) \
5146   do { \
5147     PetscAssert(sizeof(extension__), comm, PETSC_ERR_PLIB, "Zero-size extension: %s", extension__); \
5148     /* don't count the null-terminator at the end */ \
5149     const size_t ext_len = sizeof(extension__) - 1; \
5150     if (len < ext_len) { \
5151       is_extension__ = PETSC_FALSE; \
5152     } else { \
5153       PetscCall(PetscStrncmp(filename + len - ext_len, extension__, ext_len, &is_extension__)); \
5154     } \
5155   } while (0)
5156 
5157   CheckExtension(extGmsh, isGmsh);
5158   CheckExtension(extGmsh2, isGmsh2);
5159   CheckExtension(extGmsh4, isGmsh4);
5160   CheckExtension(extCGNS, isCGNS);
5161   CheckExtension(extExodus, isExodus);
5162   if (!isExodus) CheckExtension(extExodus_e, isExodus);
5163   CheckExtension(extGenesis, isGenesis);
5164   CheckExtension(extFluent, isFluent);
5165   CheckExtension(extHDF5, isHDF5);
5166   CheckExtension(extMed, isMed);
5167   CheckExtension(extPLY, isPLY);
5168   CheckExtension(extEGADSLite, isEGADSLite);
5169   CheckExtension(extEGADS, isEGADS);
5170   CheckExtension(extIGES, isIGES);
5171   CheckExtension(extSTEP, isSTEP);
5172   CheckExtension(extCV, isCV);
5173 
5174 #undef CheckExtension
5175 
5176   if (isGmsh || isGmsh2 || isGmsh4) {
5177     PetscCall(DMPlexCreateGmshFromFile(comm, filename, interpolate, dm));
5178   } else if (isCGNS) {
5179     PetscCall(DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm));
5180   } else if (isExodus || isGenesis) {
5181     PetscCall(DMPlexCreateExodusFromFile(comm, filename, interpolate, dm));
5182   } else if (isFluent) {
5183     PetscCall(DMPlexCreateFluentFromFile(comm, filename, interpolate, dm));
5184   } else if (isHDF5) {
5185     PetscBool   load_hdf5_xdmf = PETSC_FALSE;
5186     PetscViewer viewer;
5187 
5188     /* PETSC_VIEWER_HDF5_XDMF is used if the filename ends with .xdmf.h5, or if -dm_plex_create_from_hdf5_xdmf option is present */
5189     PetscCall(PetscStrncmp(&filename[PetscMax(0, len - 8)], ".xdmf", 5, &load_hdf5_xdmf));
5190     PetscCall(PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &load_hdf5_xdmf, NULL));
5191     PetscCall(PetscViewerCreate(comm, &viewer));
5192     PetscCall(PetscViewerSetType(viewer, PETSCVIEWERHDF5));
5193     PetscCall(PetscViewerSetOptionsPrefix(viewer, "dm_plex_create_"));
5194     PetscCall(PetscViewerSetFromOptions(viewer));
5195     PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
5196     PetscCall(PetscViewerFileSetName(viewer, filename));
5197 
5198     PetscCall(DMCreate(comm, dm));
5199     PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5200     PetscCall(DMSetType(*dm, DMPLEX));
5201     if (load_hdf5_xdmf) PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF));
5202     PetscCall(DMLoad(*dm, viewer));
5203     if (load_hdf5_xdmf) PetscCall(PetscViewerPopFormat(viewer));
5204     PetscCall(PetscViewerDestroy(&viewer));
5205 
5206     if (interpolate) {
5207       DM idm;
5208 
5209       PetscCall(DMPlexInterpolate(*dm, &idm));
5210       PetscCall(DMDestroy(dm));
5211       *dm = idm;
5212     }
5213   } else if (isMed) {
5214     PetscCall(DMPlexCreateMedFromFile(comm, filename, interpolate, dm));
5215   } else if (isPLY) {
5216     PetscCall(DMPlexCreatePLYFromFile(comm, filename, interpolate, dm));
5217   } else if (isEGADSLite || isEGADS || isIGES || isSTEP) {
5218     if (isEGADSLite) PetscCall(DMPlexCreateEGADSLiteFromFile(comm, filename, dm));
5219     else PetscCall(DMPlexCreateEGADSFromFile(comm, filename, dm));
5220     if (!interpolate) {
5221       DM udm;
5222 
5223       PetscCall(DMPlexUninterpolate(*dm, &udm));
5224       PetscCall(DMDestroy(dm));
5225       *dm = udm;
5226     }
5227   } else if (isCV) {
5228     PetscCall(DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm));
5229   } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
5230   PetscCall(PetscStrlen(plexname, &len));
5231   if (len) PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5232   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5233   PetscFunctionReturn(0);
5234 }
5235 /*@C
5236   DMPlexCreateEphemeral - This takes a `DMPlexTransform` and a base `DMPlex` and produces an ephemeral `DM`, meaning one that is created on the fly in response to queries.
5237 
5238   Input Parameter:
5239 . tr - The `DMPlexTransform`
5240 
5241   Output Parameter:
5242 . dm - The `DM`
5243 
5244   Notes:
5245   An emphemeral mesh is one that is not stored concretely, as in the default Plex implementation, but rather is produced on the fly in response to queries, using information from the transform and the base mesh.
5246 
5247   Level: beginner
5248 
5249 .seealso: `DMPlexCreateFromFile`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
5250 @*/
5251 PetscErrorCode DMPlexCreateEphemeral(DMPlexTransform tr, DM *dm)
5252 {
5253   DM       bdm;
5254   PetscInt Nl;
5255 
5256   PetscFunctionBegin;
5257   PetscCall(DMCreate(PetscObjectComm((PetscObject)tr), dm));
5258   PetscCall(DMSetType(*dm, DMPLEX));
5259   PetscCall(DMSetFromOptions(*dm));
5260 
5261   PetscCall(PetscObjectReference((PetscObject)tr));
5262   PetscCall(DMPlexTransformDestroy(&((DM_Plex *)(*dm)->data)->tr));
5263   ((DM_Plex *)(*dm)->data)->tr = tr;
5264 
5265   PetscCall(DMPlexTransformGetDM(tr, &bdm));
5266   PetscCall(DMGetNumLabels(bdm, &Nl));
5267   for (PetscInt l = 0; l < Nl; ++l) {
5268     DMLabel     label, labelNew;
5269     const char *lname;
5270     PetscBool   isDepth, isCellType;
5271 
5272     PetscCall(DMGetLabelName(bdm, l, &lname));
5273     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
5274     if (isDepth) continue;
5275     PetscCall(PetscStrcmp(lname, "celltype", &isCellType));
5276     if (isCellType) continue;
5277     PetscCall(DMCreateLabel(*dm, lname));
5278     PetscCall(DMGetLabel(bdm, lname, &label));
5279     PetscCall(DMGetLabel(*dm, lname, &labelNew));
5280     PetscCall(DMLabelSetType(labelNew, DMLABELEPHEMERAL));
5281     PetscCall(DMLabelEphemeralSetLabel(labelNew, label));
5282     PetscCall(DMLabelEphemeralSetTransform(labelNew, tr));
5283     PetscCall(DMLabelSetUp(labelNew));
5284   }
5285   PetscFunctionReturn(0);
5286 }
5287