xref: /petsc/src/dm/impls/plex/plexcreate.c (revision 5f06a3dd92e34bdfe6112242961ce57bf1b830e2)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h> /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>   /*I   "petscdmplex.h"   I*/
4 #include <petscsf.h>
5 #include <petscdmplextransform.h>
6 #include <petscdmlabelephemeral.h>
7 #include <petsc/private/kernels/blockmatmult.h>
8 #include <petsc/private/kernels/blockinvert.h>
9 
10 PetscLogEvent DMPLEX_CreateFromFile, DMPLEX_BuildFromCellList, DMPLEX_BuildCoordinatesFromCellList;
11 
12 /* External function declarations here */
13 static PetscErrorCode DMInitialize_Plex(DM dm);
14 
15 /* This copies internal things in the Plex structure that we generally want when making a new, related Plex */
16 PetscErrorCode DMPlexCopy_Internal(DM dmin, PetscBool copyPeriodicity, PetscBool copyOverlap, DM dmout)
17 {
18   const PetscReal         *maxCell, *Lstart, *L;
19   PetscBool                dist;
20   DMPlexReorderDefaultFlag reorder;
21 
22   PetscFunctionBegin;
23   if (copyPeriodicity) {
24     PetscCall(DMGetPeriodicity(dmin, &maxCell, &Lstart, &L));
25     PetscCall(DMSetPeriodicity(dmout, maxCell, Lstart, L));
26   }
27   PetscCall(DMPlexDistributeGetDefault(dmin, &dist));
28   PetscCall(DMPlexDistributeSetDefault(dmout, dist));
29   PetscCall(DMPlexReorderGetDefault(dmin, &reorder));
30   PetscCall(DMPlexReorderSetDefault(dmout, reorder));
31   ((DM_Plex *)dmout->data)->useHashLocation = ((DM_Plex *)dmin->data)->useHashLocation;
32   if (copyOverlap) PetscCall(DMPlexSetOverlap_Plex(dmout, dmin, 0));
33   PetscFunctionReturn(0);
34 }
35 
36 /* Replace dm with the contents of ndm, and then destroy ndm
37    - Share the DM_Plex structure
38    - Share the coordinates
39    - Share the SF
40 */
41 PetscErrorCode DMPlexReplace_Internal(DM dm, DM *ndm)
42 {
43   PetscSF          sf;
44   DM               dmNew = *ndm, coordDM, coarseDM;
45   Vec              coords;
46   const PetscReal *maxCell, *Lstart, *L;
47   PetscInt         dim, cdim;
48 
49   PetscFunctionBegin;
50   if (dm == dmNew) {
51     PetscCall(DMDestroy(ndm));
52     PetscFunctionReturn(0);
53   }
54   dm->setupcalled = dmNew->setupcalled;
55   PetscCall(DMGetDimension(dmNew, &dim));
56   PetscCall(DMSetDimension(dm, dim));
57   PetscCall(DMGetCoordinateDim(dmNew, &cdim));
58   PetscCall(DMSetCoordinateDim(dm, cdim));
59   PetscCall(DMGetPointSF(dmNew, &sf));
60   PetscCall(DMSetPointSF(dm, sf));
61   PetscCall(DMGetCoordinateDM(dmNew, &coordDM));
62   PetscCall(DMGetCoordinatesLocal(dmNew, &coords));
63   PetscCall(DMSetCoordinateDM(dm, coordDM));
64   PetscCall(DMSetCoordinatesLocal(dm, coords));
65   PetscCall(DMGetCellCoordinateDM(dmNew, &coordDM));
66   PetscCall(DMGetCellCoordinatesLocal(dmNew, &coords));
67   PetscCall(DMSetCellCoordinateDM(dm, coordDM));
68   PetscCall(DMSetCellCoordinatesLocal(dm, coords));
69   /* Do not want to create the coordinate field if it does not already exist, so do not call DMGetCoordinateField() */
70   PetscCall(DMFieldDestroy(&dm->coordinates[0].field));
71   dm->coordinates[0].field            = dmNew->coordinates[0].field;
72   ((DM_Plex *)dmNew->data)->coordFunc = ((DM_Plex *)dm->data)->coordFunc;
73   PetscCall(DMGetPeriodicity(dmNew, &maxCell, &Lstart, &L));
74   PetscCall(DMSetPeriodicity(dm, maxCell, Lstart, L));
75   PetscCall(DMDestroy_Plex(dm));
76   PetscCall(DMInitialize_Plex(dm));
77   dm->data = dmNew->data;
78   ((DM_Plex *)dmNew->data)->refct++;
79   PetscCall(DMPlexGetIsoperiodicFaceSF(dm, &sf));
80   PetscCall(DMPlexSetIsoperiodicFaceSF(dm, sf)); // for the compose function effect on dm
81   PetscCall(DMDestroyLabelLinkList_Internal(dm));
82   PetscCall(DMCopyLabels(dmNew, dm, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL));
83   PetscCall(DMGetCoarseDM(dmNew, &coarseDM));
84   PetscCall(DMSetCoarseDM(dm, coarseDM));
85   PetscCall(DMDestroy(ndm));
86   PetscFunctionReturn(0);
87 }
88 
89 /* Swap dm with the contents of dmNew
90    - Swap the DM_Plex structure
91    - Swap the coordinates
92    - Swap the point PetscSF
93 */
94 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
95 {
96   DM          coordDMA, coordDMB;
97   Vec         coordsA, coordsB;
98   PetscSF     sfA, sfB;
99   DMField     fieldTmp;
100   void       *tmp;
101   DMLabelLink listTmp;
102   DMLabel     depthTmp;
103   PetscInt    tmpI;
104 
105   PetscFunctionBegin;
106   if (dmA == dmB) PetscFunctionReturn(0);
107   PetscCall(DMGetPointSF(dmA, &sfA));
108   PetscCall(DMGetPointSF(dmB, &sfB));
109   PetscCall(PetscObjectReference((PetscObject)sfA));
110   PetscCall(DMSetPointSF(dmA, sfB));
111   PetscCall(DMSetPointSF(dmB, sfA));
112   PetscCall(PetscObjectDereference((PetscObject)sfA));
113 
114   PetscCall(DMGetCoordinateDM(dmA, &coordDMA));
115   PetscCall(DMGetCoordinateDM(dmB, &coordDMB));
116   PetscCall(PetscObjectReference((PetscObject)coordDMA));
117   PetscCall(DMSetCoordinateDM(dmA, coordDMB));
118   PetscCall(DMSetCoordinateDM(dmB, coordDMA));
119   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
120 
121   PetscCall(DMGetCoordinatesLocal(dmA, &coordsA));
122   PetscCall(DMGetCoordinatesLocal(dmB, &coordsB));
123   PetscCall(PetscObjectReference((PetscObject)coordsA));
124   PetscCall(DMSetCoordinatesLocal(dmA, coordsB));
125   PetscCall(DMSetCoordinatesLocal(dmB, coordsA));
126   PetscCall(PetscObjectDereference((PetscObject)coordsA));
127 
128   PetscCall(DMGetCellCoordinateDM(dmA, &coordDMA));
129   PetscCall(DMGetCellCoordinateDM(dmB, &coordDMB));
130   PetscCall(PetscObjectReference((PetscObject)coordDMA));
131   PetscCall(DMSetCellCoordinateDM(dmA, coordDMB));
132   PetscCall(DMSetCellCoordinateDM(dmB, coordDMA));
133   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
134 
135   PetscCall(DMGetCellCoordinatesLocal(dmA, &coordsA));
136   PetscCall(DMGetCellCoordinatesLocal(dmB, &coordsB));
137   PetscCall(PetscObjectReference((PetscObject)coordsA));
138   PetscCall(DMSetCellCoordinatesLocal(dmA, coordsB));
139   PetscCall(DMSetCellCoordinatesLocal(dmB, coordsA));
140   PetscCall(PetscObjectDereference((PetscObject)coordsA));
141 
142   fieldTmp                  = dmA->coordinates[0].field;
143   dmA->coordinates[0].field = dmB->coordinates[0].field;
144   dmB->coordinates[0].field = fieldTmp;
145   fieldTmp                  = dmA->coordinates[1].field;
146   dmA->coordinates[1].field = dmB->coordinates[1].field;
147   dmB->coordinates[1].field = fieldTmp;
148   tmp                       = dmA->data;
149   dmA->data                 = dmB->data;
150   dmB->data                 = tmp;
151   listTmp                   = dmA->labels;
152   dmA->labels               = dmB->labels;
153   dmB->labels               = listTmp;
154   depthTmp                  = dmA->depthLabel;
155   dmA->depthLabel           = dmB->depthLabel;
156   dmB->depthLabel           = depthTmp;
157   depthTmp                  = dmA->celltypeLabel;
158   dmA->celltypeLabel        = dmB->celltypeLabel;
159   dmB->celltypeLabel        = depthTmp;
160   tmpI                      = dmA->levelup;
161   dmA->levelup              = dmB->levelup;
162   dmB->levelup              = tmpI;
163   PetscFunctionReturn(0);
164 }
165 
166 PetscErrorCode DMPlexInterpolateInPlace_Internal(DM dm)
167 {
168   DM idm;
169 
170   PetscFunctionBegin;
171   PetscCall(DMPlexInterpolate(dm, &idm));
172   PetscCall(DMPlexCopyCoordinates(dm, idm));
173   PetscCall(DMPlexReplace_Internal(dm, &idm));
174   PetscFunctionReturn(0);
175 }
176 
177 /*@C
178   DMPlexCreateCoordinateSpace - Creates a finite element space for the coordinates
179 
180   Collective on dm
181 
182   Input Parameters:
183 + DM        - The DM
184 . degree    - The degree of the finite element or PETSC_DECIDE
185 - coordFunc - An optional function to map new points from refinement to the surface
186 
187   Level: advanced
188 
189 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `PetscPointFunc`, `PetscFECreateLagrange()`, `DMGetCoordinateDM()`
190 @*/
191 PetscErrorCode DMPlexCreateCoordinateSpace(DM dm, PetscInt degree, PetscPointFunc coordFunc)
192 {
193   DM_Plex     *mesh = (DM_Plex *)dm->data;
194   DM           cdm;
195   PetscDS      cds;
196   PetscFE      fe;
197   PetscClassId id;
198 
199   PetscFunctionBegin;
200   PetscCall(DMGetCoordinateDM(dm, &cdm));
201   PetscCall(DMGetDS(cdm, &cds));
202   PetscCall(PetscDSGetDiscretization(cds, 0, (PetscObject *)&fe));
203   PetscCall(PetscObjectGetClassId((PetscObject)fe, &id));
204   if (id != PETSCFE_CLASSID) {
205     PetscInt dim, dE, qorder;
206 
207     PetscCall(DMGetDimension(dm, &dim));
208     PetscCall(DMGetCoordinateDim(dm, &dE));
209     qorder = degree;
210     PetscObjectOptionsBegin((PetscObject)cdm);
211     PetscCall(PetscOptionsBoundedInt("-default_quadrature_order", "Quadrature order is one less than quadrature points per edge", "DMPlexCreateCoordinateSpace", qorder, &qorder, NULL, 0));
212     PetscOptionsEnd();
213     if (degree == PETSC_DECIDE) fe = NULL;
214     else {
215       DMPolytopeType ct;
216       PetscInt       cStart, cEnd, ctTmp;
217 
218       PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
219       // Want to match cell types
220       if (cEnd > cStart) PetscCall(DMPlexGetCellType(dm, cStart, &ct));
221       else ct = DM_POLYTOPE_UNKNOWN;
222       ctTmp = (PetscInt)ct;
223       PetscCallMPI(MPI_Allreduce(MPI_IN_PLACE, &ctTmp, 1, MPIU_INT, MPI_MIN, PetscObjectComm((PetscObject)dm)));
224       ct = (DMPolytopeType)ctTmp;
225       // Work around current bug
226       if (ct == DM_POLYTOPE_SEG_PRISM_TENSOR || ct == DM_POLYTOPE_TRI_PRISM_TENSOR || ct == DM_POLYTOPE_QUAD_PRISM_TENSOR) fe = NULL;
227       else PetscCall(PetscFECreateLagrangeByCell(PETSC_COMM_SELF, dim, dE, ct, degree, qorder, &fe));
228     }
229     if (fe) PetscCall(DMProjectCoordinates(dm, fe));
230     PetscCall(PetscFEDestroy(&fe));
231   }
232   mesh->coordFunc = coordFunc;
233   PetscFunctionReturn(0);
234 }
235 
236 /*@
237   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
238 
239   Collective
240 
241   Input Parameters:
242 + comm - The communicator for the `DM` object
243 . dim - The spatial dimension
244 . simplex - Flag for simplicial cells, otherwise they are tensor product cells
245 . interpolate - Flag to create intermediate mesh pieces (edges, faces)
246 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
247 
248   Output Parameter:
249 . dm - The `DM` object
250 
251   Level: beginner
252 
253 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMSetType()`, `DMCreate()`
254 @*/
255 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscReal refinementLimit, DM *newdm)
256 {
257   DM          dm;
258   PetscMPIInt rank;
259 
260   PetscFunctionBegin;
261   PetscCall(DMCreate(comm, &dm));
262   PetscCall(DMSetType(dm, DMPLEX));
263   PetscCall(DMSetDimension(dm, dim));
264   PetscCallMPI(MPI_Comm_rank(comm, &rank));
265   switch (dim) {
266   case 2:
267     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "triangular"));
268     else PetscCall(PetscObjectSetName((PetscObject)dm, "quadrilateral"));
269     break;
270   case 3:
271     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "tetrahedral"));
272     else PetscCall(PetscObjectSetName((PetscObject)dm, "hexahedral"));
273     break;
274   default:
275     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
276   }
277   if (rank) {
278     PetscInt numPoints[2] = {0, 0};
279     PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL));
280   } else {
281     switch (dim) {
282     case 2:
283       if (simplex) {
284         PetscInt    numPoints[2]        = {4, 2};
285         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
286         PetscInt    cones[6]            = {2, 3, 4, 5, 4, 3};
287         PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
288         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
289 
290         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
291       } else {
292         PetscInt    numPoints[2]        = {6, 2};
293         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
294         PetscInt    cones[8]            = {2, 3, 4, 5, 3, 6, 7, 4};
295         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
296         PetscScalar vertexCoords[12]    = {-1.0, -0.5, 0.0, -0.5, 0.0, 0.5, -1.0, 0.5, 1.0, -0.5, 1.0, 0.5};
297 
298         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
299       }
300       break;
301     case 3:
302       if (simplex) {
303         PetscInt    numPoints[2]        = {5, 2};
304         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
305         PetscInt    cones[8]            = {4, 3, 5, 2, 5, 3, 4, 6};
306         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
307         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0};
308 
309         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
310       } else {
311         PetscInt    numPoints[2]         = {12, 2};
312         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
313         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9, 5, 4, 10, 11, 7, 12, 13, 8};
314         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
315         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5, -1.0, 0.5, -0.5, 0.0, 0.5, -0.5, 0.0, -0.5, -0.5, -1.0, -0.5, 0.5, 0.0, -0.5, 0.5, 0.0, 0.5, 0.5, -1.0, 0.5, 0.5, 1.0, 0.5, -0.5, 1.0, -0.5, -0.5, 1.0, -0.5, 0.5, 1.0, 0.5, 0.5};
316 
317         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
318       }
319       break;
320     default:
321       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
322     }
323   }
324   *newdm = dm;
325   if (refinementLimit > 0.0) {
326     DM          rdm;
327     const char *name;
328 
329     PetscCall(DMPlexSetRefinementUniform(*newdm, PETSC_FALSE));
330     PetscCall(DMPlexSetRefinementLimit(*newdm, refinementLimit));
331     PetscCall(DMRefine(*newdm, comm, &rdm));
332     PetscCall(PetscObjectGetName((PetscObject)*newdm, &name));
333     PetscCall(PetscObjectSetName((PetscObject)rdm, name));
334     PetscCall(DMDestroy(newdm));
335     *newdm = rdm;
336   }
337   if (interpolate) {
338     DM idm;
339 
340     PetscCall(DMPlexInterpolate(*newdm, &idm));
341     PetscCall(DMDestroy(newdm));
342     *newdm = idm;
343   }
344   PetscFunctionReturn(0);
345 }
346 
347 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
348 {
349   const PetscInt numVertices    = 2;
350   PetscInt       markerRight    = 1;
351   PetscInt       markerLeft     = 1;
352   PetscBool      markerSeparate = PETSC_FALSE;
353   Vec            coordinates;
354   PetscSection   coordSection;
355   PetscScalar   *coords;
356   PetscInt       coordSize;
357   PetscMPIInt    rank;
358   PetscInt       cdim = 1, v;
359 
360   PetscFunctionBegin;
361   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
362   if (markerSeparate) {
363     markerRight = 2;
364     markerLeft  = 1;
365   }
366   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
367   if (rank == 0) {
368     PetscCall(DMPlexSetChart(dm, 0, numVertices));
369     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
370     PetscCall(DMSetLabelValue(dm, "marker", 0, markerLeft));
371     PetscCall(DMSetLabelValue(dm, "marker", 1, markerRight));
372   }
373   PetscCall(DMPlexSymmetrize(dm));
374   PetscCall(DMPlexStratify(dm));
375   /* Build coordinates */
376   PetscCall(DMSetCoordinateDim(dm, cdim));
377   PetscCall(DMGetCoordinateSection(dm, &coordSection));
378   PetscCall(PetscSectionSetNumFields(coordSection, 1));
379   PetscCall(PetscSectionSetChart(coordSection, 0, numVertices));
380   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
381   for (v = 0; v < numVertices; ++v) {
382     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
383     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
384   }
385   PetscCall(PetscSectionSetUp(coordSection));
386   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
387   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
388   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
389   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
390   PetscCall(VecSetBlockSize(coordinates, cdim));
391   PetscCall(VecSetType(coordinates, VECSTANDARD));
392   PetscCall(VecGetArray(coordinates, &coords));
393   coords[0] = lower[0];
394   coords[1] = upper[0];
395   PetscCall(VecRestoreArray(coordinates, &coords));
396   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
397   PetscCall(VecDestroy(&coordinates));
398   PetscFunctionReturn(0);
399 }
400 
401 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
402 {
403   const PetscInt numVertices    = (edges[0] + 1) * (edges[1] + 1);
404   const PetscInt numEdges       = edges[0] * (edges[1] + 1) + (edges[0] + 1) * edges[1];
405   PetscInt       markerTop      = 1;
406   PetscInt       markerBottom   = 1;
407   PetscInt       markerRight    = 1;
408   PetscInt       markerLeft     = 1;
409   PetscBool      markerSeparate = PETSC_FALSE;
410   Vec            coordinates;
411   PetscSection   coordSection;
412   PetscScalar   *coords;
413   PetscInt       coordSize;
414   PetscMPIInt    rank;
415   PetscInt       v, vx, vy;
416 
417   PetscFunctionBegin;
418   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
419   if (markerSeparate) {
420     markerTop    = 3;
421     markerBottom = 1;
422     markerRight  = 2;
423     markerLeft   = 4;
424   }
425   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
426   if (rank == 0) {
427     PetscInt e, ex, ey;
428 
429     PetscCall(DMPlexSetChart(dm, 0, numEdges + numVertices));
430     for (e = 0; e < numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
431     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
432     for (vx = 0; vx <= edges[0]; vx++) {
433       for (ey = 0; ey < edges[1]; ey++) {
434         PetscInt edge   = vx * edges[1] + ey + edges[0] * (edges[1] + 1);
435         PetscInt vertex = ey * (edges[0] + 1) + vx + numEdges;
436         PetscInt cone[2];
437 
438         cone[0] = vertex;
439         cone[1] = vertex + edges[0] + 1;
440         PetscCall(DMPlexSetCone(dm, edge, cone));
441         if (vx == edges[0]) {
442           PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
443           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
444           if (ey == edges[1] - 1) {
445             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
446             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerRight));
447           }
448         } else if (vx == 0) {
449           PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
450           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
451           if (ey == edges[1] - 1) {
452             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
453             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft));
454           }
455         }
456       }
457     }
458     for (vy = 0; vy <= edges[1]; vy++) {
459       for (ex = 0; ex < edges[0]; ex++) {
460         PetscInt edge   = vy * edges[0] + ex;
461         PetscInt vertex = vy * (edges[0] + 1) + ex + numEdges;
462         PetscInt cone[2];
463 
464         cone[0] = vertex;
465         cone[1] = vertex + 1;
466         PetscCall(DMPlexSetCone(dm, edge, cone));
467         if (vy == edges[1]) {
468           PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
469           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
470           if (ex == edges[0] - 1) {
471             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
472             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerTop));
473           }
474         } else if (vy == 0) {
475           PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
476           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
477           if (ex == edges[0] - 1) {
478             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
479             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom));
480           }
481         }
482       }
483     }
484   }
485   PetscCall(DMPlexSymmetrize(dm));
486   PetscCall(DMPlexStratify(dm));
487   /* Build coordinates */
488   PetscCall(DMSetCoordinateDim(dm, 2));
489   PetscCall(DMGetCoordinateSection(dm, &coordSection));
490   PetscCall(PetscSectionSetNumFields(coordSection, 1));
491   PetscCall(PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices));
492   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 2));
493   for (v = numEdges; v < numEdges + numVertices; ++v) {
494     PetscCall(PetscSectionSetDof(coordSection, v, 2));
495     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 2));
496   }
497   PetscCall(PetscSectionSetUp(coordSection));
498   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
499   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
500   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
501   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
502   PetscCall(VecSetBlockSize(coordinates, 2));
503   PetscCall(VecSetType(coordinates, VECSTANDARD));
504   PetscCall(VecGetArray(coordinates, &coords));
505   for (vy = 0; vy <= edges[1]; ++vy) {
506     for (vx = 0; vx <= edges[0]; ++vx) {
507       coords[(vy * (edges[0] + 1) + vx) * 2 + 0] = lower[0] + ((upper[0] - lower[0]) / edges[0]) * vx;
508       coords[(vy * (edges[0] + 1) + vx) * 2 + 1] = lower[1] + ((upper[1] - lower[1]) / edges[1]) * vy;
509     }
510   }
511   PetscCall(VecRestoreArray(coordinates, &coords));
512   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
513   PetscCall(VecDestroy(&coordinates));
514   PetscFunctionReturn(0);
515 }
516 
517 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
518 {
519   PetscInt     vertices[3], numVertices;
520   PetscInt     numFaces       = 2 * faces[0] * faces[1] + 2 * faces[1] * faces[2] + 2 * faces[0] * faces[2];
521   PetscInt     markerTop      = 1;
522   PetscInt     markerBottom   = 1;
523   PetscInt     markerFront    = 1;
524   PetscInt     markerBack     = 1;
525   PetscInt     markerRight    = 1;
526   PetscInt     markerLeft     = 1;
527   PetscBool    markerSeparate = PETSC_FALSE;
528   Vec          coordinates;
529   PetscSection coordSection;
530   PetscScalar *coords;
531   PetscInt     coordSize;
532   PetscMPIInt  rank;
533   PetscInt     v, vx, vy, vz;
534   PetscInt     voffset, iface = 0, cone[4];
535 
536   PetscFunctionBegin;
537   PetscCheck(faces[0] >= 1 && faces[1] >= 1 && faces[2] >= 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
538   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
539   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
540   if (markerSeparate) {
541     markerBottom = 1;
542     markerTop    = 2;
543     markerFront  = 3;
544     markerBack   = 4;
545     markerRight  = 5;
546     markerLeft   = 6;
547   }
548   vertices[0] = faces[0] + 1;
549   vertices[1] = faces[1] + 1;
550   vertices[2] = faces[2] + 1;
551   numVertices = vertices[0] * vertices[1] * vertices[2];
552   if (rank == 0) {
553     PetscInt f;
554 
555     PetscCall(DMPlexSetChart(dm, 0, numFaces + numVertices));
556     for (f = 0; f < numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
557     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
558 
559     /* Side 0 (Top) */
560     for (vy = 0; vy < faces[1]; vy++) {
561       for (vx = 0; vx < faces[0]; vx++) {
562         voffset = numFaces + vertices[0] * vertices[1] * (vertices[2] - 1) + vy * vertices[0] + vx;
563         cone[0] = voffset;
564         cone[1] = voffset + 1;
565         cone[2] = voffset + vertices[0] + 1;
566         cone[3] = voffset + vertices[0];
567         PetscCall(DMPlexSetCone(dm, iface, cone));
568         PetscCall(DMSetLabelValue(dm, "marker", iface, markerTop));
569         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerTop));
570         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerTop));
571         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerTop));
572         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerTop));
573         iface++;
574       }
575     }
576 
577     /* Side 1 (Bottom) */
578     for (vy = 0; vy < faces[1]; vy++) {
579       for (vx = 0; vx < faces[0]; vx++) {
580         voffset = numFaces + vy * (faces[0] + 1) + vx;
581         cone[0] = voffset + 1;
582         cone[1] = voffset;
583         cone[2] = voffset + vertices[0];
584         cone[3] = voffset + vertices[0] + 1;
585         PetscCall(DMPlexSetCone(dm, iface, cone));
586         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBottom));
587         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBottom));
588         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBottom));
589         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerBottom));
590         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerBottom));
591         iface++;
592       }
593     }
594 
595     /* Side 2 (Front) */
596     for (vz = 0; vz < faces[2]; vz++) {
597       for (vx = 0; vx < faces[0]; vx++) {
598         voffset = numFaces + vz * vertices[0] * vertices[1] + vx;
599         cone[0] = voffset;
600         cone[1] = voffset + 1;
601         cone[2] = voffset + vertices[0] * vertices[1] + 1;
602         cone[3] = voffset + vertices[0] * vertices[1];
603         PetscCall(DMPlexSetCone(dm, iface, cone));
604         PetscCall(DMSetLabelValue(dm, "marker", iface, markerFront));
605         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerFront));
606         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerFront));
607         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerFront));
608         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerFront));
609         iface++;
610       }
611     }
612 
613     /* Side 3 (Back) */
614     for (vz = 0; vz < faces[2]; vz++) {
615       for (vx = 0; vx < faces[0]; vx++) {
616         voffset = numFaces + vz * vertices[0] * vertices[1] + vertices[0] * (vertices[1] - 1) + vx;
617         cone[0] = voffset + vertices[0] * vertices[1];
618         cone[1] = voffset + vertices[0] * vertices[1] + 1;
619         cone[2] = voffset + 1;
620         cone[3] = voffset;
621         PetscCall(DMPlexSetCone(dm, iface, cone));
622         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBack));
623         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBack));
624         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBack));
625         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerBack));
626         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerBack));
627         iface++;
628       }
629     }
630 
631     /* Side 4 (Left) */
632     for (vz = 0; vz < faces[2]; vz++) {
633       for (vy = 0; vy < faces[1]; vy++) {
634         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0];
635         cone[0] = voffset;
636         cone[1] = voffset + vertices[0] * vertices[1];
637         cone[2] = voffset + vertices[0] * vertices[1] + vertices[0];
638         cone[3] = voffset + vertices[0];
639         PetscCall(DMPlexSetCone(dm, iface, cone));
640         PetscCall(DMSetLabelValue(dm, "marker", iface, markerLeft));
641         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerLeft));
642         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerLeft));
643         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[1] + 0, markerLeft));
644         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerLeft));
645         iface++;
646       }
647     }
648 
649     /* Side 5 (Right) */
650     for (vz = 0; vz < faces[2]; vz++) {
651       for (vy = 0; vy < faces[1]; vy++) {
652         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0] + faces[0];
653         cone[0] = voffset + vertices[0] * vertices[1];
654         cone[1] = voffset;
655         cone[2] = voffset + vertices[0];
656         cone[3] = voffset + vertices[0] * vertices[1] + vertices[0];
657         PetscCall(DMPlexSetCone(dm, iface, cone));
658         PetscCall(DMSetLabelValue(dm, "marker", iface, markerRight));
659         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerRight));
660         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerRight));
661         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerRight));
662         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerRight));
663         iface++;
664       }
665     }
666   }
667   PetscCall(DMPlexSymmetrize(dm));
668   PetscCall(DMPlexStratify(dm));
669   /* Build coordinates */
670   PetscCall(DMSetCoordinateDim(dm, 3));
671   PetscCall(DMGetCoordinateSection(dm, &coordSection));
672   PetscCall(PetscSectionSetNumFields(coordSection, 1));
673   PetscCall(PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices));
674   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 3));
675   for (v = numFaces; v < numFaces + numVertices; ++v) {
676     PetscCall(PetscSectionSetDof(coordSection, v, 3));
677     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 3));
678   }
679   PetscCall(PetscSectionSetUp(coordSection));
680   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
681   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
682   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
683   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
684   PetscCall(VecSetBlockSize(coordinates, 3));
685   PetscCall(VecSetType(coordinates, VECSTANDARD));
686   PetscCall(VecGetArray(coordinates, &coords));
687   for (vz = 0; vz <= faces[2]; ++vz) {
688     for (vy = 0; vy <= faces[1]; ++vy) {
689       for (vx = 0; vx <= faces[0]; ++vx) {
690         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 0] = lower[0] + ((upper[0] - lower[0]) / faces[0]) * vx;
691         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 1] = lower[1] + ((upper[1] - lower[1]) / faces[1]) * vy;
692         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 2] = lower[2] + ((upper[2] - lower[2]) / faces[2]) * vz;
693       }
694     }
695   }
696   PetscCall(VecRestoreArray(coordinates, &coords));
697   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
698   PetscCall(VecDestroy(&coordinates));
699   PetscFunctionReturn(0);
700 }
701 
702 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate)
703 {
704   PetscFunctionBegin;
705   PetscValidLogicalCollectiveInt(dm, dim, 2);
706   PetscCall(DMSetDimension(dm, dim - 1));
707   PetscCall(DMSetCoordinateDim(dm, dim));
708   switch (dim) {
709   case 1:
710     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(dm, lower, upper, faces));
711     break;
712   case 2:
713     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(dm, lower, upper, faces));
714     break;
715   case 3:
716     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(dm, lower, upper, faces));
717     break;
718   default:
719     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Dimension not supported: %" PetscInt_FMT, dim);
720   }
721   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
722   PetscFunctionReturn(0);
723 }
724 
725 /*@C
726   DMPlexCreateBoxSurfaceMesh - Creates a mesh on the surface of the tensor product of unit intervals (box) using tensor cells (hexahedra).
727 
728   Collective
729 
730   Input Parameters:
731 + comm        - The communicator for the `DM` object
732 . dim         - The spatial dimension of the box, so the resulting mesh is has dimension dim-1
733 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
734 . lower       - The lower left corner, or NULL for (0, 0, 0)
735 . upper       - The upper right corner, or NULL for (1, 1, 1)
736 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
737 
738   Output Parameter:
739 . dm  - The `DM` object
740 
741   Level: beginner
742 
743 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateBoxMesh()`, `DMPlexCreateFromFile()`, `DMSetType()`, `DMCreate()`
744 @*/
745 PetscErrorCode DMPlexCreateBoxSurfaceMesh(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate, DM *dm)
746 {
747   PetscInt  fac[3] = {1, 1, 1};
748   PetscReal low[3] = {0, 0, 0};
749   PetscReal upp[3] = {1, 1, 1};
750 
751   PetscFunctionBegin;
752   PetscCall(DMCreate(comm, dm));
753   PetscCall(DMSetType(*dm, DMPLEX));
754   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(*dm, dim, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, interpolate));
755   PetscFunctionReturn(0);
756 }
757 
758 static PetscErrorCode DMPlexCreateLineMesh_Internal(DM dm, PetscInt segments, PetscReal lower, PetscReal upper, DMBoundaryType bd)
759 {
760   PetscInt     i, fStart, fEnd, numCells = 0, numVerts = 0;
761   PetscInt     numPoints[2], *coneSize, *cones, *coneOrientations;
762   PetscScalar *vertexCoords;
763   PetscReal    L, maxCell;
764   PetscBool    markerSeparate = PETSC_FALSE;
765   PetscInt     markerLeft = 1, faceMarkerLeft = 1;
766   PetscInt     markerRight = 1, faceMarkerRight = 2;
767   PetscBool    wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
768   PetscMPIInt  rank;
769 
770   PetscFunctionBegin;
771   PetscValidPointer(dm, 1);
772 
773   PetscCall(DMSetDimension(dm, 1));
774   PetscCall(DMCreateLabel(dm, "marker"));
775   PetscCall(DMCreateLabel(dm, "Face Sets"));
776 
777   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
778   if (rank == 0) numCells = segments;
779   if (rank == 0) numVerts = segments + (wrap ? 0 : 1);
780 
781   numPoints[0] = numVerts;
782   numPoints[1] = numCells;
783   PetscCall(PetscMalloc4(numCells + numVerts, &coneSize, numCells * 2, &cones, numCells + numVerts, &coneOrientations, numVerts, &vertexCoords));
784   PetscCall(PetscArrayzero(coneOrientations, numCells + numVerts));
785   for (i = 0; i < numCells; ++i) coneSize[i] = 2;
786   for (i = 0; i < numVerts; ++i) coneSize[numCells + i] = 0;
787   for (i = 0; i < numCells; ++i) {
788     cones[2 * i]     = numCells + i % numVerts;
789     cones[2 * i + 1] = numCells + (i + 1) % numVerts;
790   }
791   for (i = 0; i < numVerts; ++i) vertexCoords[i] = lower + (upper - lower) * ((PetscReal)i / (PetscReal)numCells);
792   PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
793   PetscCall(PetscFree4(coneSize, cones, coneOrientations, vertexCoords));
794 
795   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
796   if (markerSeparate) {
797     markerLeft  = faceMarkerLeft;
798     markerRight = faceMarkerRight;
799   }
800   if (!wrap && rank == 0) {
801     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
802     PetscCall(DMSetLabelValue(dm, "marker", fStart, markerLeft));
803     PetscCall(DMSetLabelValue(dm, "marker", fEnd - 1, markerRight));
804     PetscCall(DMSetLabelValue(dm, "Face Sets", fStart, faceMarkerLeft));
805     PetscCall(DMSetLabelValue(dm, "Face Sets", fEnd - 1, faceMarkerRight));
806   }
807   if (wrap) {
808     L       = upper - lower;
809     maxCell = (PetscReal)1.1 * (L / (PetscReal)PetscMax(1, segments));
810     PetscCall(DMSetPeriodicity(dm, &maxCell, &lower, &L));
811   }
812   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
813   PetscFunctionReturn(0);
814 }
815 
816 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
817 {
818   DM      boundary, vol;
819   DMLabel bdlabel;
820 
821   PetscFunctionBegin;
822   PetscValidPointer(dm, 1);
823   for (PetscInt i = 0; i < dim; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
824   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &boundary));
825   PetscCall(DMSetType(boundary, DMPLEX));
826   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(boundary, dim, faces, lower, upper, PETSC_FALSE));
827   PetscCall(DMPlexGenerate(boundary, NULL, interpolate, &vol));
828   PetscCall(DMGetLabel(vol, "marker", &bdlabel));
829   if (bdlabel) PetscCall(DMPlexLabelComplete(vol, bdlabel));
830   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_FALSE, vol));
831   PetscCall(DMPlexReplace_Internal(dm, &vol));
832   PetscCall(DMDestroy(&boundary));
833   PetscFunctionReturn(0);
834 }
835 
836 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
837 {
838   DMLabel     cutLabel  = NULL;
839   PetscInt    markerTop = 1, faceMarkerTop = 1;
840   PetscInt    markerBottom = 1, faceMarkerBottom = 1;
841   PetscInt    markerFront = 1, faceMarkerFront = 1;
842   PetscInt    markerBack = 1, faceMarkerBack = 1;
843   PetscInt    markerRight = 1, faceMarkerRight = 1;
844   PetscInt    markerLeft = 1, faceMarkerLeft = 1;
845   PetscInt    dim;
846   PetscBool   markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
847   PetscMPIInt rank;
848 
849   PetscFunctionBegin;
850   PetscCall(DMGetDimension(dm, &dim));
851   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
852   PetscCall(DMCreateLabel(dm, "marker"));
853   PetscCall(DMCreateLabel(dm, "Face Sets"));
854   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
855   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST || bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST || bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
856     if (cutMarker) {
857       PetscCall(DMCreateLabel(dm, "periodic_cut"));
858       PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
859     }
860   }
861   switch (dim) {
862   case 2:
863     faceMarkerTop    = 3;
864     faceMarkerBottom = 1;
865     faceMarkerRight  = 2;
866     faceMarkerLeft   = 4;
867     break;
868   case 3:
869     faceMarkerBottom = 1;
870     faceMarkerTop    = 2;
871     faceMarkerFront  = 3;
872     faceMarkerBack   = 4;
873     faceMarkerRight  = 5;
874     faceMarkerLeft   = 6;
875     break;
876   default:
877     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
878   }
879   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
880   if (markerSeparate) {
881     markerBottom = faceMarkerBottom;
882     markerTop    = faceMarkerTop;
883     markerFront  = faceMarkerFront;
884     markerBack   = faceMarkerBack;
885     markerRight  = faceMarkerRight;
886     markerLeft   = faceMarkerLeft;
887   }
888   {
889     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
890     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
891     const PetscInt numZEdges    = rank == 0 ? edges[2] : 0;
892     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0] + 1) : 0;
893     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1] + 1) : 0;
894     const PetscInt numZVertices = rank == 0 ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2] + 1) : 0;
895     const PetscInt numCells     = numXEdges * numYEdges * numZEdges;
896     const PetscInt numXFaces    = numYEdges * numZEdges;
897     const PetscInt numYFaces    = numXEdges * numZEdges;
898     const PetscInt numZFaces    = numXEdges * numYEdges;
899     const PetscInt numTotXFaces = numXVertices * numXFaces;
900     const PetscInt numTotYFaces = numYVertices * numYFaces;
901     const PetscInt numTotZFaces = numZVertices * numZFaces;
902     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
903     const PetscInt numTotXEdges = numXEdges * numYVertices * numZVertices;
904     const PetscInt numTotYEdges = numYEdges * numXVertices * numZVertices;
905     const PetscInt numTotZEdges = numZEdges * numXVertices * numYVertices;
906     const PetscInt numVertices  = numXVertices * numYVertices * numZVertices;
907     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
908     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
909     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
910     const PetscInt firstYFace   = firstXFace + numTotXFaces;
911     const PetscInt firstZFace   = firstYFace + numTotYFaces;
912     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
913     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
914     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
915     Vec            coordinates;
916     PetscSection   coordSection;
917     PetscScalar   *coords;
918     PetscInt       coordSize;
919     PetscInt       v, vx, vy, vz;
920     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
921 
922     PetscCall(DMPlexSetChart(dm, 0, numCells + numFaces + numEdges + numVertices));
923     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
924     for (f = firstXFace; f < firstXFace + numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
925     for (e = firstXEdge; e < firstXEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
926     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
927     /* Build cells */
928     for (fz = 0; fz < numZEdges; ++fz) {
929       for (fy = 0; fy < numYEdges; ++fy) {
930         for (fx = 0; fx < numXEdges; ++fx) {
931           PetscInt cell  = (fz * numYEdges + fy) * numXEdges + fx;
932           PetscInt faceB = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
933           PetscInt faceT = firstZFace + (fy * numXEdges + fx) * numZVertices + ((fz + 1) % numZVertices);
934           PetscInt faceF = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
935           PetscInt faceK = firstYFace + (fz * numXEdges + fx) * numYVertices + ((fy + 1) % numYVertices);
936           PetscInt faceL = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
937           PetscInt faceR = firstXFace + (fz * numYEdges + fy) * numXVertices + ((fx + 1) % numXVertices);
938           /* B,  T,  F,  K,  R,  L */
939           PetscInt ornt[6] = {-2, 0, 0, -3, 0, -2}; /* ??? */
940           PetscInt cone[6];
941 
942           /* no boundary twisting in 3D */
943           cone[0] = faceB;
944           cone[1] = faceT;
945           cone[2] = faceF;
946           cone[3] = faceK;
947           cone[4] = faceR;
948           cone[5] = faceL;
949           PetscCall(DMPlexSetCone(dm, cell, cone));
950           PetscCall(DMPlexSetConeOrientation(dm, cell, ornt));
951           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
952           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
953           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
954         }
955       }
956     }
957     /* Build x faces */
958     for (fz = 0; fz < numZEdges; ++fz) {
959       for (fy = 0; fy < numYEdges; ++fy) {
960         for (fx = 0; fx < numXVertices; ++fx) {
961           PetscInt face    = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
962           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
963           PetscInt edgeR   = firstZEdge + (((fy + 1) % numYVertices) * numXVertices + fx) * numZEdges + fz;
964           PetscInt edgeB   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
965           PetscInt edgeT   = firstYEdge + (((fz + 1) % numZVertices) * numXVertices + fx) * numYEdges + fy;
966           PetscInt ornt[4] = {0, 0, -1, -1};
967           PetscInt cone[4];
968 
969           if (dim == 3) {
970             /* markers */
971             if (bdX != DM_BOUNDARY_PERIODIC) {
972               if (fx == numXVertices - 1) {
973                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight));
974                 PetscCall(DMSetLabelValue(dm, "marker", face, markerRight));
975               } else if (fx == 0) {
976                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft));
977                 PetscCall(DMSetLabelValue(dm, "marker", face, markerLeft));
978               }
979             }
980           }
981           cone[0] = edgeB;
982           cone[1] = edgeR;
983           cone[2] = edgeT;
984           cone[3] = edgeL;
985           PetscCall(DMPlexSetCone(dm, face, cone));
986           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
987         }
988       }
989     }
990     /* Build y faces */
991     for (fz = 0; fz < numZEdges; ++fz) {
992       for (fx = 0; fx < numXEdges; ++fx) {
993         for (fy = 0; fy < numYVertices; ++fy) {
994           PetscInt face    = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
995           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
996           PetscInt edgeR   = firstZEdge + (fy * numXVertices + ((fx + 1) % numXVertices)) * numZEdges + fz;
997           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
998           PetscInt edgeT   = firstXEdge + (((fz + 1) % numZVertices) * numYVertices + fy) * numXEdges + fx;
999           PetscInt ornt[4] = {0, 0, -1, -1};
1000           PetscInt cone[4];
1001 
1002           if (dim == 3) {
1003             /* markers */
1004             if (bdY != DM_BOUNDARY_PERIODIC) {
1005               if (fy == numYVertices - 1) {
1006                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack));
1007                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBack));
1008               } else if (fy == 0) {
1009                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront));
1010                 PetscCall(DMSetLabelValue(dm, "marker", face, markerFront));
1011               }
1012             }
1013           }
1014           cone[0] = edgeB;
1015           cone[1] = edgeR;
1016           cone[2] = edgeT;
1017           cone[3] = edgeL;
1018           PetscCall(DMPlexSetCone(dm, face, cone));
1019           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1020         }
1021       }
1022     }
1023     /* Build z faces */
1024     for (fy = 0; fy < numYEdges; ++fy) {
1025       for (fx = 0; fx < numXEdges; ++fx) {
1026         for (fz = 0; fz < numZVertices; fz++) {
1027           PetscInt face    = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
1028           PetscInt edgeL   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
1029           PetscInt edgeR   = firstYEdge + (fz * numXVertices + ((fx + 1) % numXVertices)) * numYEdges + fy;
1030           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1031           PetscInt edgeT   = firstXEdge + (fz * numYVertices + ((fy + 1) % numYVertices)) * numXEdges + fx;
1032           PetscInt ornt[4] = {0, 0, -1, -1};
1033           PetscInt cone[4];
1034 
1035           if (dim == 2) {
1036             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges - 1) {
1037               edgeR += numYEdges - 1 - 2 * fy;
1038               ornt[1] = -1;
1039             }
1040             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges - 1) {
1041               edgeT += numXEdges - 1 - 2 * fx;
1042               ornt[2] = 0;
1043             }
1044             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1045             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1046           } else {
1047             /* markers */
1048             if (bdZ != DM_BOUNDARY_PERIODIC) {
1049               if (fz == numZVertices - 1) {
1050                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop));
1051                 PetscCall(DMSetLabelValue(dm, "marker", face, markerTop));
1052               } else if (fz == 0) {
1053                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom));
1054                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBottom));
1055               }
1056             }
1057           }
1058           cone[0] = edgeB;
1059           cone[1] = edgeR;
1060           cone[2] = edgeT;
1061           cone[3] = edgeL;
1062           PetscCall(DMPlexSetCone(dm, face, cone));
1063           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1064         }
1065       }
1066     }
1067     /* Build Z edges*/
1068     for (vy = 0; vy < numYVertices; vy++) {
1069       for (vx = 0; vx < numXVertices; vx++) {
1070         for (ez = 0; ez < numZEdges; ez++) {
1071           const PetscInt edge    = firstZEdge + (vy * numXVertices + vx) * numZEdges + ez;
1072           const PetscInt vertexB = firstVertex + (ez * numYVertices + vy) * numXVertices + vx;
1073           const PetscInt vertexT = firstVertex + (((ez + 1) % numZVertices) * numYVertices + vy) * numXVertices + vx;
1074           PetscInt       cone[2];
1075 
1076           cone[0] = vertexB;
1077           cone[1] = vertexT;
1078           PetscCall(DMPlexSetCone(dm, edge, cone));
1079           if (dim == 3) {
1080             if (bdX != DM_BOUNDARY_PERIODIC) {
1081               if (vx == numXVertices - 1) {
1082                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1083                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1084                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1085               } else if (vx == 0) {
1086                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1087                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1088                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1089               }
1090             }
1091             if (bdY != DM_BOUNDARY_PERIODIC) {
1092               if (vy == numYVertices - 1) {
1093                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1094                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1095                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1096               } else if (vy == 0) {
1097                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1098                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1099                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1100               }
1101             }
1102           }
1103         }
1104       }
1105     }
1106     /* Build Y edges*/
1107     for (vz = 0; vz < numZVertices; vz++) {
1108       for (vx = 0; vx < numXVertices; vx++) {
1109         for (ey = 0; ey < numYEdges; ey++) {
1110           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges - 1) ? (numXVertices - vx - 1) : (vz * numYVertices + ((ey + 1) % numYVertices)) * numXVertices + vx;
1111           const PetscInt edge    = firstYEdge + (vz * numXVertices + vx) * numYEdges + ey;
1112           const PetscInt vertexF = firstVertex + (vz * numYVertices + ey) * numXVertices + vx;
1113           const PetscInt vertexK = firstVertex + nextv;
1114           PetscInt       cone[2];
1115 
1116           cone[0] = vertexF;
1117           cone[1] = vertexK;
1118           PetscCall(DMPlexSetCone(dm, edge, cone));
1119           if (dim == 2) {
1120             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1121               if (vx == numXVertices - 1) {
1122                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight));
1123                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1124                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1125                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1126               } else if (vx == 0) {
1127                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft));
1128                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1129                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1130                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1131               }
1132             } else {
1133               if (vx == 0 && cutLabel) {
1134                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1135                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1136                 if (ey == numYEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1137               }
1138             }
1139           } else {
1140             if (bdX != DM_BOUNDARY_PERIODIC) {
1141               if (vx == numXVertices - 1) {
1142                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1143                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1144                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1145               } else if (vx == 0) {
1146                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1147                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1148                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1149               }
1150             }
1151             if (bdZ != DM_BOUNDARY_PERIODIC) {
1152               if (vz == numZVertices - 1) {
1153                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1154                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1155                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1156               } else if (vz == 0) {
1157                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1158                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1159                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1160               }
1161             }
1162           }
1163         }
1164       }
1165     }
1166     /* Build X edges*/
1167     for (vz = 0; vz < numZVertices; vz++) {
1168       for (vy = 0; vy < numYVertices; vy++) {
1169         for (ex = 0; ex < numXEdges; ex++) {
1170           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges - 1) ? (numYVertices - vy - 1) * numXVertices : (vz * numYVertices + vy) * numXVertices + (ex + 1) % numXVertices;
1171           const PetscInt edge    = firstXEdge + (vz * numYVertices + vy) * numXEdges + ex;
1172           const PetscInt vertexL = firstVertex + (vz * numYVertices + vy) * numXVertices + ex;
1173           const PetscInt vertexR = firstVertex + nextv;
1174           PetscInt       cone[2];
1175 
1176           cone[0] = vertexL;
1177           cone[1] = vertexR;
1178           PetscCall(DMPlexSetCone(dm, edge, cone));
1179           if (dim == 2) {
1180             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1181               if (vy == numYVertices - 1) {
1182                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop));
1183                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1184                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1185                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1186               } else if (vy == 0) {
1187                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom));
1188                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1189                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1190                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1191               }
1192             } else {
1193               if (vy == 0 && cutLabel) {
1194                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1195                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1196                 if (ex == numXEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1197               }
1198             }
1199           } else {
1200             if (bdY != DM_BOUNDARY_PERIODIC) {
1201               if (vy == numYVertices - 1) {
1202                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1203                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1204                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1205               } else if (vy == 0) {
1206                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1207                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1208                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1209               }
1210             }
1211             if (bdZ != DM_BOUNDARY_PERIODIC) {
1212               if (vz == numZVertices - 1) {
1213                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1214                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1215                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1216               } else if (vz == 0) {
1217                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1218                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1219                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1220               }
1221             }
1222           }
1223         }
1224       }
1225     }
1226     PetscCall(DMPlexSymmetrize(dm));
1227     PetscCall(DMPlexStratify(dm));
1228     /* Build coordinates */
1229     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1230     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1231     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1232     PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVertices));
1233     for (v = firstVertex; v < firstVertex + numVertices; ++v) {
1234       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1235       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1236     }
1237     PetscCall(PetscSectionSetUp(coordSection));
1238     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1239     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1240     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1241     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1242     PetscCall(VecSetBlockSize(coordinates, dim));
1243     PetscCall(VecSetType(coordinates, VECSTANDARD));
1244     PetscCall(VecGetArray(coordinates, &coords));
1245     for (vz = 0; vz < numZVertices; ++vz) {
1246       for (vy = 0; vy < numYVertices; ++vy) {
1247         for (vx = 0; vx < numXVertices; ++vx) {
1248           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * vx;
1249           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * vy;
1250           if (dim == 3) coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 2] = lower[2] + ((upper[2] - lower[2]) / numZEdges) * vz;
1251         }
1252       }
1253     }
1254     PetscCall(VecRestoreArray(coordinates, &coords));
1255     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1256     PetscCall(VecDestroy(&coordinates));
1257   }
1258   PetscFunctionReturn(0);
1259 }
1260 
1261 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1262 {
1263   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1264   PetscInt       fac[3] = {0, 0, 0}, d;
1265 
1266   PetscFunctionBegin;
1267   PetscValidPointer(dm, 1);
1268   PetscValidLogicalCollectiveInt(dm, dim, 2);
1269   PetscCall(DMSetDimension(dm, dim));
1270   for (d = 0; d < dim; ++d) {
1271     fac[d] = faces[d];
1272     bdt[d] = periodicity[d];
1273   }
1274   PetscCall(DMPlexCreateCubeMesh_Internal(dm, lower, upper, fac, bdt[0], bdt[1], bdt[2]));
1275   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST || periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST || (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
1276     PetscReal L[3]       = {-1., -1., 0.};
1277     PetscReal maxCell[3] = {-1., -1., 0.};
1278 
1279     for (d = 0; d < dim; ++d) {
1280       if (periodicity[d] != DM_BOUNDARY_NONE) {
1281         L[d]       = upper[d] - lower[d];
1282         maxCell[d] = 1.1 * (L[d] / PetscMax(1, faces[d]));
1283       }
1284     }
1285     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1286   }
1287   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
1288   PetscFunctionReturn(0);
1289 }
1290 
1291 static PetscErrorCode DMPlexCreateBoxMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1292 {
1293   PetscBool box_sfc = PETSC_FALSE;
1294   PetscFunctionBegin;
1295   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_box_sfc", &box_sfc, NULL));
1296   if (box_sfc) PetscCall(DMPlexCreateBoxMesh_Tensor_SFC_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1297   else if (dim == 1) PetscCall(DMPlexCreateLineMesh_Internal(dm, faces[0], lower[0], upper[0], periodicity[0]));
1298   else if (simplex) PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1299   else PetscCall(DMPlexCreateBoxMesh_Tensor_Internal(dm, dim, faces, lower, upper, periodicity));
1300   if (!interpolate && dim > 1 && !simplex) {
1301     DM udm;
1302 
1303     PetscCall(DMPlexUninterpolate(dm, &udm));
1304     PetscCall(DMPlexCopyCoordinates(dm, udm));
1305     PetscCall(DMPlexReplace_Internal(dm, &udm));
1306   }
1307   PetscFunctionReturn(0);
1308 }
1309 
1310 /*@C
1311   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
1312 
1313   Collective
1314 
1315   Input Parameters:
1316 + comm        - The communicator for the `DM` object
1317 . dim         - The spatial dimension
1318 . simplex     - `PETSC_TRUE` for simplices, `PETSC_FALSE` for tensor cells
1319 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1320 . lower       - The lower left corner, or NULL for (0, 0, 0)
1321 . upper       - The upper right corner, or NULL for (1, 1, 1)
1322 . periodicity - The boundary type for the X,Y,Z direction, or NULL for `DM_BOUNDARY_NONE`
1323 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1324 
1325   Output Parameter:
1326 . dm  - The `DM` object
1327 
1328   Level: beginner
1329 
1330   Note:
1331    To customize this mesh using options, use
1332 .vb
1333   DMCreate(comm, &dm);
1334   DMSetType(dm, DMPLEX);
1335   DMSetFromOptions(dm);
1336 .ve
1337 and use the options in `DMSetFromOptions()`.
1338 
1339   Here is the numbering returned for 2 faces in each direction for tensor cells:
1340 .vb
1341  10---17---11---18----12
1342   |         |         |
1343   |         |         |
1344  20    2   22    3    24
1345   |         |         |
1346   |         |         |
1347   7---15----8---16----9
1348   |         |         |
1349   |         |         |
1350  19    0   21    1   23
1351   |         |         |
1352   |         |         |
1353   4---13----5---14----6
1354 .ve
1355 and for simplicial cells
1356 .vb
1357  14----8---15----9----16
1358   |\     5  |\      7 |
1359   | \       | \       |
1360  13   2    14    3    15
1361   | 4   \   | 6   \   |
1362   |       \ |       \ |
1363  11----6---12----7----13
1364   |\        |\        |
1365   | \    1  | \     3 |
1366  10   0    11    1    12
1367   | 0   \   | 2   \   |
1368   |       \ |       \ |
1369   8----4----9----5----10
1370 .ve
1371 
1372 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
1373 @*/
1374 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
1375 {
1376   PetscInt       fac[3] = {1, 1, 1};
1377   PetscReal      low[3] = {0, 0, 0};
1378   PetscReal      upp[3] = {1, 1, 1};
1379   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1380 
1381   PetscFunctionBegin;
1382   PetscCall(DMCreate(comm, dm));
1383   PetscCall(DMSetType(*dm, DMPLEX));
1384   PetscCall(DMPlexCreateBoxMesh_Internal(*dm, dim, simplex, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt, interpolate));
1385   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1386   PetscFunctionReturn(0);
1387 }
1388 
1389 static PetscErrorCode DMPlexCreateWedgeBoxMesh_Internal(DM dm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1390 {
1391   DM       bdm, vol;
1392   PetscInt i;
1393 
1394   PetscFunctionBegin;
1395   for (i = 0; i < 3; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity not yet supported");
1396   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &bdm));
1397   PetscCall(DMSetType(bdm, DMPLEX));
1398   PetscCall(DMSetDimension(bdm, 2));
1399   PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(bdm, 2, faces, lower, upper, periodicity, PETSC_TRUE));
1400   PetscCall(DMPlexExtrude(bdm, faces[2], upper[2] - lower[2], PETSC_TRUE, PETSC_FALSE, NULL, NULL, &vol));
1401   PetscCall(DMDestroy(&bdm));
1402   PetscCall(DMPlexReplace_Internal(dm, &vol));
1403   if (lower[2] != 0.0) {
1404     Vec          v;
1405     PetscScalar *x;
1406     PetscInt     cDim, n;
1407 
1408     PetscCall(DMGetCoordinatesLocal(dm, &v));
1409     PetscCall(VecGetBlockSize(v, &cDim));
1410     PetscCall(VecGetLocalSize(v, &n));
1411     PetscCall(VecGetArray(v, &x));
1412     x += cDim;
1413     for (i = 0; i < n; i += cDim) x[i] += lower[2];
1414     PetscCall(VecRestoreArray(v, &x));
1415     PetscCall(DMSetCoordinatesLocal(dm, v));
1416   }
1417   PetscFunctionReturn(0);
1418 }
1419 
1420 /*@
1421   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tesselating the (x,y) plane and extruding in the third direction using wedge cells.
1422 
1423   Collective
1424 
1425   Input Parameters:
1426 + comm        - The communicator for the `DM` object
1427 . faces       - Number of faces per dimension, or NULL for (1, 1, 1)
1428 . lower       - The lower left corner, or NULL for (0, 0, 0)
1429 . upper       - The upper right corner, or NULL for (1, 1, 1)
1430 . periodicity - The boundary type for the X,Y,Z direction, or NULL for `DM_BOUNDARY_NONE`
1431 . orderHeight - If `PETSC_TRUE`, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1432 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1433 
1434   Output Parameter:
1435 . dm  - The `DM` object
1436 
1437   Level: beginner
1438 
1439 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateWedgeCylinderMesh()`, `DMExtrude()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1440 @*/
1441 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool orderHeight, PetscBool interpolate, DM *dm)
1442 {
1443   PetscInt       fac[3] = {1, 1, 1};
1444   PetscReal      low[3] = {0, 0, 0};
1445   PetscReal      upp[3] = {1, 1, 1};
1446   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1447 
1448   PetscFunctionBegin;
1449   PetscCall(DMCreate(comm, dm));
1450   PetscCall(DMSetType(*dm, DMPLEX));
1451   PetscCall(DMPlexCreateWedgeBoxMesh_Internal(*dm, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt));
1452   if (!interpolate) {
1453     DM udm;
1454 
1455     PetscCall(DMPlexUninterpolate(*dm, &udm));
1456     PetscCall(DMPlexReplace_Internal(*dm, &udm));
1457   }
1458   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1459   PetscFunctionReturn(0);
1460 }
1461 
1462 /*
1463   DMPlexTensorPointLexicographic_Private - Returns all tuples of size 'len' with nonnegative integers that are all less than or equal to 'max' for that dimension.
1464                                            Ordering is lexicographic with lowest index as least significant in ordering.
1465                                            e.g. for len == 2 and max == 2, this will return, in order, {0,0}, {1,0}, {2,0}, {0,1}, {1,1}, {2,1}, {0,2}, {1,2}, {2,2}.
1466 
1467   Input Parameters:
1468 + len - The length of the tuple
1469 . max - The maximum for each dimension, so values are in [0, max)
1470 - tup - A tuple of length len+1: tup[len] > 0 indicates a stopping condition
1471 
1472   Output Parameter:
1473 . tup - A tuple of len integers whose entries are at most 'max'
1474 
1475   Level: developer
1476 
1477 .seealso: PetscDualSpaceTensorPointLexicographic_Internal(), PetscDualSpaceLatticePointLexicographic_Internal()
1478 */
1479 static PetscErrorCode DMPlexTensorPointLexicographic_Private(PetscInt len, const PetscInt max[], PetscInt tup[])
1480 {
1481   PetscInt i;
1482 
1483   PetscFunctionBegin;
1484   for (i = 0; i < len; ++i) {
1485     if (tup[i] < max[i] - 1) {
1486       break;
1487     } else {
1488       tup[i] = 0;
1489     }
1490   }
1491   if (i == len) tup[i - 1] = max[i - 1];
1492   else ++tup[i];
1493   PetscFunctionReturn(0);
1494 }
1495 
1496 static PetscInt TupleToIndex_Private(PetscInt len, const PetscInt max[], const PetscInt tup[])
1497 {
1498   PetscInt i, idx = tup[len - 1];
1499 
1500   for (i = len - 2; i >= 0; --i) {
1501     idx *= max[i];
1502     idx += tup[i];
1503   }
1504   return idx;
1505 }
1506 
1507 static PetscErrorCode DestroyExtent_Private(void *extent)
1508 {
1509   return PetscFree(extent);
1510 }
1511 
1512 static PetscErrorCode DMPlexCreateHypercubicMesh_Internal(DM dm, PetscInt dim, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], const DMBoundaryType bd[])
1513 {
1514   Vec          coordinates;
1515   PetscSection coordSection;
1516   DMLabel      cutLabel    = NULL;
1517   PetscBool    cutMarker   = PETSC_FALSE;
1518   PetscBool    periodic    = PETSC_FALSE;
1519   PetscInt     numCells    = 1, c;
1520   PetscInt     numVertices = 1, v;
1521   PetscScalar *coords;
1522   PetscInt    *vertices, *vert, *vtmp, *supp, cone[2];
1523   PetscInt     d, e, cell = 0, coordSize;
1524   PetscMPIInt  rank;
1525 
1526   PetscFunctionBegin;
1527   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1528   PetscCall(DMSetDimension(dm, dim));
1529   PetscCall(PetscCalloc4(dim, &vertices, dim, &vert, dim, &vtmp, 2 * dim, &supp));
1530   PetscCall(DMCreateLabel(dm, "marker"));
1531   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
1532   for (d = 0; d < dim; ++d) periodic = (periodic || bd[d] == DM_BOUNDARY_PERIODIC) ? PETSC_TRUE : PETSC_FALSE;
1533   if (periodic && cutMarker) {
1534     PetscCall(DMCreateLabel(dm, "periodic_cut"));
1535     PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
1536   }
1537   for (d = 0; d < dim; ++d) PetscCheck(bd[d] == DM_BOUNDARY_PERIODIC, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Hypercubic mesh must be periodic now");
1538   for (d = 0; d < dim; ++d) {
1539     vertices[d] = edges[d];
1540     numVertices *= vertices[d];
1541   }
1542   numCells = numVertices * dim;
1543   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1544   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, 2));
1545   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetSupportSize(dm, v, 2 * dim));
1546   /* TODO Loop over boundary and reset support sizes */
1547   PetscCall(DMSetUp(dm)); /* Allocate space for cones and supports */
1548   /* Build cell cones and vertex supports */
1549   PetscCall(DMCreateLabel(dm, "celltype"));
1550   while (vert[dim - 1] < vertices[dim - 1]) {
1551     const PetscInt vertex = TupleToIndex_Private(dim, vertices, vert) + numCells;
1552     PetscInt       s      = 0;
1553 
1554     PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT ":", vertex);
1555     for (d = 0; d < dim; ++d) PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, vert[d]);
1556     PetscPrintf(PETSC_COMM_SELF, "\n");
1557     PetscCall(DMPlexSetCellType(dm, vertex, DM_POLYTOPE_POINT));
1558     for (d = 0; d < dim; ++d) {
1559       for (e = 0; e < dim; ++e) vtmp[e] = vert[e];
1560       vtmp[d] = (vert[d] + 1) % vertices[d];
1561       cone[0] = vertex;
1562       cone[1] = TupleToIndex_Private(dim, vertices, vtmp) + numCells;
1563       PetscPrintf(PETSC_COMM_SELF, "  Vertex %" PetscInt_FMT ":", cone[1]);
1564       for (e = 0; e < dim; ++e) PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, vtmp[e]);
1565       PetscPrintf(PETSC_COMM_SELF, "\n");
1566       PetscCall(DMPlexSetCone(dm, cell, cone));
1567       PetscCall(DMPlexSetCellType(dm, cell, DM_POLYTOPE_SEGMENT));
1568       PetscPrintf(PETSC_COMM_SELF, "  Edge %" PetscInt_FMT " (%" PetscInt_FMT " %" PetscInt_FMT ")\n", cell, cone[0], cone[1]);
1569       ++cell;
1570     }
1571     for (d = 0; d < dim; ++d) {
1572       for (e = 0; e < dim; ++e) vtmp[e] = vert[e];
1573       vtmp[d]   = (vert[d] + vertices[d] - 1) % vertices[d];
1574       supp[s++] = TupleToIndex_Private(dim, vertices, vtmp) * dim + d;
1575       supp[s++] = (vertex - numCells) * dim + d;
1576       PetscCall(DMPlexSetSupport(dm, vertex, supp));
1577     }
1578     PetscCall(DMPlexTensorPointLexicographic_Private(dim, vertices, vert));
1579   }
1580   PetscCall(DMPlexStratify(dm));
1581   /* Build coordinates */
1582   PetscCall(DMGetCoordinateSection(dm, &coordSection));
1583   PetscCall(PetscSectionSetNumFields(coordSection, 1));
1584   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1585   PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1586   for (v = numCells; v < numCells + numVertices; ++v) {
1587     PetscCall(PetscSectionSetDof(coordSection, v, dim));
1588     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1589   }
1590   PetscCall(PetscSectionSetUp(coordSection));
1591   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1592   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1593   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1594   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1595   PetscCall(VecSetBlockSize(coordinates, dim));
1596   PetscCall(VecSetType(coordinates, VECSTANDARD));
1597   PetscCall(VecGetArray(coordinates, &coords));
1598   for (d = 0; d < dim; ++d) vert[d] = 0;
1599   while (vert[dim - 1] < vertices[dim - 1]) {
1600     const PetscInt vertex = TupleToIndex_Private(dim, vertices, vert);
1601 
1602     for (d = 0; d < dim; ++d) coords[vertex * dim + d] = lower[d] + ((upper[d] - lower[d]) / vertices[d]) * vert[d];
1603     PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT ":", vertex);
1604     for (d = 0; d < dim; ++d) PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, vert[d]);
1605     for (d = 0; d < dim; ++d) PetscPrintf(PETSC_COMM_SELF, " %g", (double)PetscRealPart(coords[vertex * dim + d]));
1606     PetscPrintf(PETSC_COMM_SELF, "\n");
1607     PetscCall(DMPlexTensorPointLexicographic_Private(dim, vertices, vert));
1608   }
1609   PetscCall(VecRestoreArray(coordinates, &coords));
1610   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1611   PetscCall(VecDestroy(&coordinates));
1612   PetscCall(PetscFree4(vertices, vert, vtmp, supp));
1613   //PetscCall(DMSetPeriodicity(dm, NULL, lower, upper));
1614   // Attach the extent
1615   {
1616     PetscContainer c;
1617     PetscInt      *extent;
1618 
1619     PetscCall(PetscMalloc1(dim, &extent));
1620     for (PetscInt d = 0; d < dim; ++d) extent[d] = edges[d];
1621     PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &c));
1622     PetscCall(PetscContainerSetUserDestroy(c, DestroyExtent_Private));
1623     PetscCall(PetscContainerSetPointer(c, extent));
1624     PetscCall(PetscObjectCompose((PetscObject)dm, "_extent", (PetscObject)c));
1625     PetscCall(PetscContainerDestroy(&c));
1626   }
1627   PetscFunctionReturn(0);
1628 }
1629 
1630 /*@C
1631   DMPlexCreateHypercubicMesh - Creates a peridoic mesh on the tensor product of unit intervals using only vertices and edges.
1632 
1633   Collective
1634 
1635   Input Parameters:
1636 + comm        - The communicator for the DM object
1637 . dim         - The spatial dimension
1638 . edges       - Number of edges per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1639 . lower       - The lower left corner, or NULL for (0, 0, 0)
1640 - upper       - The upper right corner, or NULL for (1, 1, 1)
1641 
1642   Output Parameter:
1643 . dm  - The DM object
1644 
1645   Note: If you want to customize this mesh using options, you just need to
1646 $  DMCreate(comm, &dm);
1647 $  DMSetType(dm, DMPLEX);
1648 $  DMSetFromOptions(dm);
1649 and use the options on the DMSetFromOptions() page.
1650 
1651   The vertices are numbered is lexicographic order, and the dim edges exiting a vertex in the positive orthant are number consecutively,
1652 $ 18--0-19--2-20--4-18
1653 $  |     |     |     |
1654 $ 13    15    17    13
1655 $  |     |     |     |
1656 $ 24-12-25-14-26-16-24
1657 $  |     |     |     |
1658 $  7     9    11     7
1659 $  |     |     |     |
1660 $ 21--6-22--8-23-10-21
1661 $  |     |     |     |
1662 $  1     3     5     1
1663 $  |     |     |     |
1664 $ 18--0-19--2-20--4-18
1665 
1666   Level: beginner
1667 
1668 .seealso: DMSetFromOptions(), DMPlexCreateFromFile(), DMPlexCreateHexCylinderMesh(), DMSetType(), DMCreate()
1669 @*/
1670 PetscErrorCode DMPlexCreateHypercubicMesh(MPI_Comm comm, PetscInt dim, const PetscInt edges[], const PetscReal lower[], const PetscReal upper[], DM *dm)
1671 {
1672   PetscInt       *edg;
1673   PetscReal      *low, *upp;
1674   DMBoundaryType *bdt;
1675   PetscInt        d;
1676 
1677   PetscFunctionBegin;
1678   PetscCall(DMCreate(comm, dm));
1679   PetscCall(DMSetType(*dm, DMPLEX));
1680   PetscCall(PetscMalloc4(dim, &edg, dim, &low, dim, &upp, dim, &bdt));
1681   for (d = 0; d < dim; ++d) {
1682     edg[d] = edges ? edges[d] : 1;
1683     low[d] = lower ? lower[d] : 0.;
1684     upp[d] = upper ? upper[d] : 1.;
1685     bdt[d] = DM_BOUNDARY_PERIODIC;
1686   }
1687   PetscCall(DMPlexCreateHypercubicMesh_Internal(*dm, dim, low, upp, edg, bdt));
1688   PetscCall(PetscFree4(edg, low, upp, bdt));
1689   PetscFunctionReturn(0);
1690 }
1691 
1692 /*@C
1693   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all `DM` options in the database.
1694 
1695   Logically Collective on dm
1696 
1697   Input Parameters:
1698 + dm - the DM context
1699 - prefix - the prefix to prepend to all option names
1700 
1701   Level: advanced
1702 
1703   Note:
1704   A hyphen (-) must NOT be given at the beginning of the prefix name.
1705   The first character of all runtime options is AUTOMATICALLY the hyphen.
1706 
1707 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `SNESSetFromOptions()`
1708 @*/
1709 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
1710 {
1711   DM_Plex *mesh = (DM_Plex *)dm->data;
1712 
1713   PetscFunctionBegin;
1714   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1715   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, prefix));
1716   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)mesh->partitioner, prefix));
1717   PetscFunctionReturn(0);
1718 }
1719 
1720 /* Remap geometry to cylinder
1721    TODO: This only works for a single refinement, then it is broken
1722 
1723      Interior square: Linear interpolation is correct
1724      The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
1725      such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
1726 
1727        phi     = arctan(y/x)
1728        d_close = sqrt(1/8 + 1/4 sin^2(phi))
1729        d_far   = sqrt(1/2 + sin^2(phi))
1730 
1731      so we remap them using
1732 
1733        x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
1734        y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
1735 
1736      If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
1737 */
1738 static void snapToCylinder(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
1739 {
1740   const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1741   const PetscReal ds2 = 0.5 * dis;
1742 
1743   if ((PetscAbsScalar(u[0]) <= ds2) && (PetscAbsScalar(u[1]) <= ds2)) {
1744     f0[0] = u[0];
1745     f0[1] = u[1];
1746   } else {
1747     PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
1748 
1749     x    = PetscRealPart(u[0]);
1750     y    = PetscRealPart(u[1]);
1751     phi  = PetscAtan2Real(y, x);
1752     sinp = PetscSinReal(phi);
1753     cosp = PetscCosReal(phi);
1754     if ((PetscAbsReal(phi) > PETSC_PI / 4.0) && (PetscAbsReal(phi) < 3.0 * PETSC_PI / 4.0)) {
1755       dc = PetscAbsReal(ds2 / sinp);
1756       df = PetscAbsReal(dis / sinp);
1757       xc = ds2 * x / PetscAbsReal(y);
1758       yc = ds2 * PetscSignReal(y);
1759     } else {
1760       dc = PetscAbsReal(ds2 / cosp);
1761       df = PetscAbsReal(dis / cosp);
1762       xc = ds2 * PetscSignReal(x);
1763       yc = ds2 * y / PetscAbsReal(x);
1764     }
1765     f0[0] = xc + (u[0] - xc) * (1.0 - dc) / (df - dc);
1766     f0[1] = yc + (u[1] - yc) * (1.0 - dc) / (df - dc);
1767   }
1768   f0[2] = u[2];
1769 }
1770 
1771 static PetscErrorCode DMPlexCreateHexCylinderMesh_Internal(DM dm, DMBoundaryType periodicZ)
1772 {
1773   const PetscInt dim = 3;
1774   PetscInt       numCells, numVertices;
1775   PetscMPIInt    rank;
1776 
1777   PetscFunctionBegin;
1778   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1779   PetscCall(DMSetDimension(dm, dim));
1780   /* Create topology */
1781   {
1782     PetscInt cone[8], c;
1783 
1784     numCells    = rank == 0 ? 5 : 0;
1785     numVertices = rank == 0 ? 16 : 0;
1786     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1787       numCells *= 3;
1788       numVertices = rank == 0 ? 24 : 0;
1789     }
1790     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1791     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 8));
1792     PetscCall(DMSetUp(dm));
1793     if (rank == 0) {
1794       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1795         cone[0] = 15;
1796         cone[1] = 18;
1797         cone[2] = 17;
1798         cone[3] = 16;
1799         cone[4] = 31;
1800         cone[5] = 32;
1801         cone[6] = 33;
1802         cone[7] = 34;
1803         PetscCall(DMPlexSetCone(dm, 0, cone));
1804         cone[0] = 16;
1805         cone[1] = 17;
1806         cone[2] = 24;
1807         cone[3] = 23;
1808         cone[4] = 32;
1809         cone[5] = 36;
1810         cone[6] = 37;
1811         cone[7] = 33; /* 22 25 26 21 */
1812         PetscCall(DMPlexSetCone(dm, 1, cone));
1813         cone[0] = 18;
1814         cone[1] = 27;
1815         cone[2] = 24;
1816         cone[3] = 17;
1817         cone[4] = 34;
1818         cone[5] = 33;
1819         cone[6] = 37;
1820         cone[7] = 38;
1821         PetscCall(DMPlexSetCone(dm, 2, cone));
1822         cone[0] = 29;
1823         cone[1] = 27;
1824         cone[2] = 18;
1825         cone[3] = 15;
1826         cone[4] = 35;
1827         cone[5] = 31;
1828         cone[6] = 34;
1829         cone[7] = 38;
1830         PetscCall(DMPlexSetCone(dm, 3, cone));
1831         cone[0] = 29;
1832         cone[1] = 15;
1833         cone[2] = 16;
1834         cone[3] = 23;
1835         cone[4] = 35;
1836         cone[5] = 36;
1837         cone[6] = 32;
1838         cone[7] = 31;
1839         PetscCall(DMPlexSetCone(dm, 4, cone));
1840 
1841         cone[0] = 31;
1842         cone[1] = 34;
1843         cone[2] = 33;
1844         cone[3] = 32;
1845         cone[4] = 19;
1846         cone[5] = 22;
1847         cone[6] = 21;
1848         cone[7] = 20;
1849         PetscCall(DMPlexSetCone(dm, 5, cone));
1850         cone[0] = 32;
1851         cone[1] = 33;
1852         cone[2] = 37;
1853         cone[3] = 36;
1854         cone[4] = 22;
1855         cone[5] = 25;
1856         cone[6] = 26;
1857         cone[7] = 21;
1858         PetscCall(DMPlexSetCone(dm, 6, cone));
1859         cone[0] = 34;
1860         cone[1] = 38;
1861         cone[2] = 37;
1862         cone[3] = 33;
1863         cone[4] = 20;
1864         cone[5] = 21;
1865         cone[6] = 26;
1866         cone[7] = 28;
1867         PetscCall(DMPlexSetCone(dm, 7, cone));
1868         cone[0] = 35;
1869         cone[1] = 38;
1870         cone[2] = 34;
1871         cone[3] = 31;
1872         cone[4] = 30;
1873         cone[5] = 19;
1874         cone[6] = 20;
1875         cone[7] = 28;
1876         PetscCall(DMPlexSetCone(dm, 8, cone));
1877         cone[0] = 35;
1878         cone[1] = 31;
1879         cone[2] = 32;
1880         cone[3] = 36;
1881         cone[4] = 30;
1882         cone[5] = 25;
1883         cone[6] = 22;
1884         cone[7] = 19;
1885         PetscCall(DMPlexSetCone(dm, 9, cone));
1886 
1887         cone[0] = 19;
1888         cone[1] = 20;
1889         cone[2] = 21;
1890         cone[3] = 22;
1891         cone[4] = 15;
1892         cone[5] = 16;
1893         cone[6] = 17;
1894         cone[7] = 18;
1895         PetscCall(DMPlexSetCone(dm, 10, cone));
1896         cone[0] = 22;
1897         cone[1] = 21;
1898         cone[2] = 26;
1899         cone[3] = 25;
1900         cone[4] = 16;
1901         cone[5] = 23;
1902         cone[6] = 24;
1903         cone[7] = 17;
1904         PetscCall(DMPlexSetCone(dm, 11, cone));
1905         cone[0] = 20;
1906         cone[1] = 28;
1907         cone[2] = 26;
1908         cone[3] = 21;
1909         cone[4] = 18;
1910         cone[5] = 17;
1911         cone[6] = 24;
1912         cone[7] = 27;
1913         PetscCall(DMPlexSetCone(dm, 12, cone));
1914         cone[0] = 30;
1915         cone[1] = 28;
1916         cone[2] = 20;
1917         cone[3] = 19;
1918         cone[4] = 29;
1919         cone[5] = 15;
1920         cone[6] = 18;
1921         cone[7] = 27;
1922         PetscCall(DMPlexSetCone(dm, 13, cone));
1923         cone[0] = 30;
1924         cone[1] = 19;
1925         cone[2] = 22;
1926         cone[3] = 25;
1927         cone[4] = 29;
1928         cone[5] = 23;
1929         cone[6] = 16;
1930         cone[7] = 15;
1931         PetscCall(DMPlexSetCone(dm, 14, cone));
1932       } else {
1933         cone[0] = 5;
1934         cone[1] = 8;
1935         cone[2] = 7;
1936         cone[3] = 6;
1937         cone[4] = 9;
1938         cone[5] = 12;
1939         cone[6] = 11;
1940         cone[7] = 10;
1941         PetscCall(DMPlexSetCone(dm, 0, cone));
1942         cone[0] = 6;
1943         cone[1] = 7;
1944         cone[2] = 14;
1945         cone[3] = 13;
1946         cone[4] = 12;
1947         cone[5] = 15;
1948         cone[6] = 16;
1949         cone[7] = 11;
1950         PetscCall(DMPlexSetCone(dm, 1, cone));
1951         cone[0] = 8;
1952         cone[1] = 17;
1953         cone[2] = 14;
1954         cone[3] = 7;
1955         cone[4] = 10;
1956         cone[5] = 11;
1957         cone[6] = 16;
1958         cone[7] = 18;
1959         PetscCall(DMPlexSetCone(dm, 2, cone));
1960         cone[0] = 19;
1961         cone[1] = 17;
1962         cone[2] = 8;
1963         cone[3] = 5;
1964         cone[4] = 20;
1965         cone[5] = 9;
1966         cone[6] = 10;
1967         cone[7] = 18;
1968         PetscCall(DMPlexSetCone(dm, 3, cone));
1969         cone[0] = 19;
1970         cone[1] = 5;
1971         cone[2] = 6;
1972         cone[3] = 13;
1973         cone[4] = 20;
1974         cone[5] = 15;
1975         cone[6] = 12;
1976         cone[7] = 9;
1977         PetscCall(DMPlexSetCone(dm, 4, cone));
1978       }
1979     }
1980     PetscCall(DMPlexSymmetrize(dm));
1981     PetscCall(DMPlexStratify(dm));
1982   }
1983   /* Create cube geometry */
1984   {
1985     Vec             coordinates;
1986     PetscSection    coordSection;
1987     PetscScalar    *coords;
1988     PetscInt        coordSize, v;
1989     const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1990     const PetscReal ds2 = dis / 2.0;
1991 
1992     /* Build coordinates */
1993     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1994     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1995     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1996     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1997     for (v = numCells; v < numCells + numVertices; ++v) {
1998       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1999       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
2000     }
2001     PetscCall(PetscSectionSetUp(coordSection));
2002     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2003     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2004     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2005     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2006     PetscCall(VecSetBlockSize(coordinates, dim));
2007     PetscCall(VecSetType(coordinates, VECSTANDARD));
2008     PetscCall(VecGetArray(coordinates, &coords));
2009     if (rank == 0) {
2010       coords[0 * dim + 0]  = -ds2;
2011       coords[0 * dim + 1]  = -ds2;
2012       coords[0 * dim + 2]  = 0.0;
2013       coords[1 * dim + 0]  = ds2;
2014       coords[1 * dim + 1]  = -ds2;
2015       coords[1 * dim + 2]  = 0.0;
2016       coords[2 * dim + 0]  = ds2;
2017       coords[2 * dim + 1]  = ds2;
2018       coords[2 * dim + 2]  = 0.0;
2019       coords[3 * dim + 0]  = -ds2;
2020       coords[3 * dim + 1]  = ds2;
2021       coords[3 * dim + 2]  = 0.0;
2022       coords[4 * dim + 0]  = -ds2;
2023       coords[4 * dim + 1]  = -ds2;
2024       coords[4 * dim + 2]  = 1.0;
2025       coords[5 * dim + 0]  = -ds2;
2026       coords[5 * dim + 1]  = ds2;
2027       coords[5 * dim + 2]  = 1.0;
2028       coords[6 * dim + 0]  = ds2;
2029       coords[6 * dim + 1]  = ds2;
2030       coords[6 * dim + 2]  = 1.0;
2031       coords[7 * dim + 0]  = ds2;
2032       coords[7 * dim + 1]  = -ds2;
2033       coords[7 * dim + 2]  = 1.0;
2034       coords[8 * dim + 0]  = dis;
2035       coords[8 * dim + 1]  = -dis;
2036       coords[8 * dim + 2]  = 0.0;
2037       coords[9 * dim + 0]  = dis;
2038       coords[9 * dim + 1]  = dis;
2039       coords[9 * dim + 2]  = 0.0;
2040       coords[10 * dim + 0] = dis;
2041       coords[10 * dim + 1] = -dis;
2042       coords[10 * dim + 2] = 1.0;
2043       coords[11 * dim + 0] = dis;
2044       coords[11 * dim + 1] = dis;
2045       coords[11 * dim + 2] = 1.0;
2046       coords[12 * dim + 0] = -dis;
2047       coords[12 * dim + 1] = dis;
2048       coords[12 * dim + 2] = 0.0;
2049       coords[13 * dim + 0] = -dis;
2050       coords[13 * dim + 1] = dis;
2051       coords[13 * dim + 2] = 1.0;
2052       coords[14 * dim + 0] = -dis;
2053       coords[14 * dim + 1] = -dis;
2054       coords[14 * dim + 2] = 0.0;
2055       coords[15 * dim + 0] = -dis;
2056       coords[15 * dim + 1] = -dis;
2057       coords[15 * dim + 2] = 1.0;
2058       if (periodicZ == DM_BOUNDARY_PERIODIC) {
2059         /* 15 31 19 */ coords[16 * dim + 0] = -ds2;
2060         coords[16 * dim + 1]                = -ds2;
2061         coords[16 * dim + 2]                = 0.5;
2062         /* 16 32 22 */ coords[17 * dim + 0] = ds2;
2063         coords[17 * dim + 1]                = -ds2;
2064         coords[17 * dim + 2]                = 0.5;
2065         /* 17 33 21 */ coords[18 * dim + 0] = ds2;
2066         coords[18 * dim + 1]                = ds2;
2067         coords[18 * dim + 2]                = 0.5;
2068         /* 18 34 20 */ coords[19 * dim + 0] = -ds2;
2069         coords[19 * dim + 1]                = ds2;
2070         coords[19 * dim + 2]                = 0.5;
2071         /* 29 35 30 */ coords[20 * dim + 0] = -dis;
2072         coords[20 * dim + 1]                = -dis;
2073         coords[20 * dim + 2]                = 0.5;
2074         /* 23 36 25 */ coords[21 * dim + 0] = dis;
2075         coords[21 * dim + 1]                = -dis;
2076         coords[21 * dim + 2]                = 0.5;
2077         /* 24 37 26 */ coords[22 * dim + 0] = dis;
2078         coords[22 * dim + 1]                = dis;
2079         coords[22 * dim + 2]                = 0.5;
2080         /* 27 38 28 */ coords[23 * dim + 0] = -dis;
2081         coords[23 * dim + 1]                = dis;
2082         coords[23 * dim + 2]                = 0.5;
2083       }
2084     }
2085     PetscCall(VecRestoreArray(coordinates, &coords));
2086     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2087     PetscCall(VecDestroy(&coordinates));
2088   }
2089   /* Create periodicity */
2090   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
2091     PetscReal L[3]       = {-1., -1., 0.};
2092     PetscReal maxCell[3] = {-1., -1., 0.};
2093     PetscReal lower[3]   = {0.0, 0.0, 0.0};
2094     PetscReal upper[3]   = {1.0, 1.0, 1.5};
2095     PetscInt  numZCells  = 3;
2096 
2097     L[2]       = upper[2] - lower[2];
2098     maxCell[2] = 1.1 * (L[2] / numZCells);
2099     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
2100   }
2101   {
2102     DM          cdm;
2103     PetscDS     cds;
2104     PetscScalar c[2] = {1.0, 1.0};
2105 
2106     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToCylinder));
2107     PetscCall(DMGetCoordinateDM(dm, &cdm));
2108     PetscCall(DMGetDS(cdm, &cds));
2109     PetscCall(PetscDSSetConstants(cds, 2, c));
2110   }
2111   /* Wait for coordinate creation before doing in-place modification */
2112   PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2113   PetscFunctionReturn(0);
2114 }
2115 
2116 /*@
2117   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
2118 
2119   Collective
2120 
2121   Input Parameters:
2122 + comm      - The communicator for the `DM` object
2123 - periodicZ - The boundary type for the Z direction
2124 
2125   Output Parameter:
2126 . dm  - The DM object
2127 
2128   Level: beginner
2129 
2130   Note:
2131   Here is the output numbering looking from the bottom of the cylinder:
2132 .vb
2133        17-----14
2134         |     |
2135         |  2  |
2136         |     |
2137  17-----8-----7-----14
2138   |     |     |     |
2139   |  3  |  0  |  1  |
2140   |     |     |     |
2141  19-----5-----6-----13
2142         |     |
2143         |  4  |
2144         |     |
2145        19-----13
2146 
2147  and up through the top
2148 
2149        18-----16
2150         |     |
2151         |  2  |
2152         |     |
2153  18----10----11-----16
2154   |     |     |     |
2155   |  3  |  0  |  1  |
2156   |     |     |     |
2157  20-----9----12-----15
2158         |     |
2159         |  4  |
2160         |     |
2161        20-----15
2162 .ve
2163 
2164 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2165 @*/
2166 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, DMBoundaryType periodicZ, DM *dm)
2167 {
2168   PetscFunctionBegin;
2169   PetscValidPointer(dm, 3);
2170   PetscCall(DMCreate(comm, dm));
2171   PetscCall(DMSetType(*dm, DMPLEX));
2172   PetscCall(DMPlexCreateHexCylinderMesh_Internal(*dm, periodicZ));
2173   PetscFunctionReturn(0);
2174 }
2175 
2176 static PetscErrorCode DMPlexCreateWedgeCylinderMesh_Internal(DM dm, PetscInt n, PetscBool interpolate)
2177 {
2178   const PetscInt dim = 3;
2179   PetscInt       numCells, numVertices, v;
2180   PetscMPIInt    rank;
2181 
2182   PetscFunctionBegin;
2183   PetscCheck(n >= 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %" PetscInt_FMT " cannot be negative", n);
2184   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2185   PetscCall(DMSetDimension(dm, dim));
2186   /* Must create the celltype label here so that we do not automatically try to compute the types */
2187   PetscCall(DMCreateLabel(dm, "celltype"));
2188   /* Create topology */
2189   {
2190     PetscInt cone[6], c;
2191 
2192     numCells    = rank == 0 ? n : 0;
2193     numVertices = rank == 0 ? 2 * (n + 1) : 0;
2194     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
2195     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
2196     PetscCall(DMSetUp(dm));
2197     for (c = 0; c < numCells; c++) {
2198       cone[0] = c + n * 1;
2199       cone[1] = (c + 1) % n + n * 1;
2200       cone[2] = 0 + 3 * n;
2201       cone[3] = c + n * 2;
2202       cone[4] = (c + 1) % n + n * 2;
2203       cone[5] = 1 + 3 * n;
2204       PetscCall(DMPlexSetCone(dm, c, cone));
2205       PetscCall(DMPlexSetCellType(dm, c, DM_POLYTOPE_TRI_PRISM_TENSOR));
2206     }
2207     PetscCall(DMPlexSymmetrize(dm));
2208     PetscCall(DMPlexStratify(dm));
2209   }
2210   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetCellType(dm, v, DM_POLYTOPE_POINT));
2211   /* Create cylinder geometry */
2212   {
2213     Vec          coordinates;
2214     PetscSection coordSection;
2215     PetscScalar *coords;
2216     PetscInt     coordSize, c;
2217 
2218     /* Build coordinates */
2219     PetscCall(DMGetCoordinateSection(dm, &coordSection));
2220     PetscCall(PetscSectionSetNumFields(coordSection, 1));
2221     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
2222     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
2223     for (v = numCells; v < numCells + numVertices; ++v) {
2224       PetscCall(PetscSectionSetDof(coordSection, v, dim));
2225       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
2226     }
2227     PetscCall(PetscSectionSetUp(coordSection));
2228     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2229     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2230     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2231     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2232     PetscCall(VecSetBlockSize(coordinates, dim));
2233     PetscCall(VecSetType(coordinates, VECSTANDARD));
2234     PetscCall(VecGetArray(coordinates, &coords));
2235     for (c = 0; c < numCells; c++) {
2236       coords[(c + 0 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
2237       coords[(c + 0 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
2238       coords[(c + 0 * n) * dim + 2] = 1.0;
2239       coords[(c + 1 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
2240       coords[(c + 1 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
2241       coords[(c + 1 * n) * dim + 2] = 0.0;
2242     }
2243     if (rank == 0) {
2244       coords[(2 * n + 0) * dim + 0] = 0.0;
2245       coords[(2 * n + 0) * dim + 1] = 0.0;
2246       coords[(2 * n + 0) * dim + 2] = 1.0;
2247       coords[(2 * n + 1) * dim + 0] = 0.0;
2248       coords[(2 * n + 1) * dim + 1] = 0.0;
2249       coords[(2 * n + 1) * dim + 2] = 0.0;
2250     }
2251     PetscCall(VecRestoreArray(coordinates, &coords));
2252     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2253     PetscCall(VecDestroy(&coordinates));
2254   }
2255   /* Interpolate */
2256   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2257   PetscFunctionReturn(0);
2258 }
2259 
2260 /*@
2261   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
2262 
2263   Collective
2264 
2265   Input Parameters:
2266 + comm - The communicator for the `DM` object
2267 . n    - The number of wedges around the origin
2268 - interpolate - Create edges and faces
2269 
2270   Output Parameter:
2271 . dm  - The `DM` object
2272 
2273   Level: beginner
2274 
2275 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2276 @*/
2277 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
2278 {
2279   PetscFunctionBegin;
2280   PetscValidPointer(dm, 4);
2281   PetscCall(DMCreate(comm, dm));
2282   PetscCall(DMSetType(*dm, DMPLEX));
2283   PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(*dm, n, interpolate));
2284   PetscFunctionReturn(0);
2285 }
2286 
2287 static inline PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2288 {
2289   PetscReal prod = 0.0;
2290   PetscInt  i;
2291   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
2292   return PetscSqrtReal(prod);
2293 }
2294 static inline PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2295 {
2296   PetscReal prod = 0.0;
2297   PetscInt  i;
2298   for (i = 0; i < dim; ++i) prod += x[i] * y[i];
2299   return prod;
2300 }
2301 
2302 /* The first constant is the sphere radius */
2303 static void snapToSphere(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
2304 {
2305   PetscReal r     = PetscRealPart(constants[0]);
2306   PetscReal norm2 = 0.0, fac;
2307   PetscInt  n     = uOff[1] - uOff[0], d;
2308 
2309   for (d = 0; d < n; ++d) norm2 += PetscSqr(PetscRealPart(u[d]));
2310   fac = r / PetscSqrtReal(norm2);
2311   for (d = 0; d < n; ++d) f0[d] = u[d] * fac;
2312 }
2313 
2314 static PetscErrorCode DMPlexCreateSphereMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, PetscReal R)
2315 {
2316   const PetscInt embedDim = dim + 1;
2317   PetscSection   coordSection;
2318   Vec            coordinates;
2319   PetscScalar   *coords;
2320   PetscReal     *coordsIn;
2321   PetscInt       numCells, numEdges, numVerts = 0, firstVertex = 0, v, firstEdge, coordSize, d, c, e;
2322   PetscMPIInt    rank;
2323 
2324   PetscFunctionBegin;
2325   PetscValidLogicalCollectiveBool(dm, simplex, 3);
2326   PetscCall(DMSetDimension(dm, dim));
2327   PetscCall(DMSetCoordinateDim(dm, dim + 1));
2328   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2329   switch (dim) {
2330   case 2:
2331     if (simplex) {
2332       const PetscReal radius    = PetscSqrtReal(1 + PETSC_PHI * PETSC_PHI) / (1.0 + PETSC_PHI);
2333       const PetscReal edgeLen   = 2.0 / (1.0 + PETSC_PHI) * (R / radius);
2334       const PetscInt  degree    = 5;
2335       PetscReal       vertex[3] = {0.0, 1.0 / (1.0 + PETSC_PHI), PETSC_PHI / (1.0 + PETSC_PHI)};
2336       PetscInt        s[3]      = {1, 1, 1};
2337       PetscInt        cone[3];
2338       PetscInt       *graph, p, i, j, k;
2339 
2340       vertex[0] *= R / radius;
2341       vertex[1] *= R / radius;
2342       vertex[2] *= R / radius;
2343       numCells    = rank == 0 ? 20 : 0;
2344       numVerts    = rank == 0 ? 12 : 0;
2345       firstVertex = numCells;
2346       /* Use icosahedron, which for a R-sphere has coordinates which are all cyclic permutations of
2347 
2348            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
2349 
2350          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2351          length is then given by 2/(1+\phi) = 2 * 0.38197 = 0.76393.
2352       */
2353       /* Construct vertices */
2354       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2355       if (rank == 0) {
2356         for (p = 0, i = 0; p < embedDim; ++p) {
2357           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2358             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2359               for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertex[(d + p) % embedDim];
2360               ++i;
2361             }
2362           }
2363         }
2364       }
2365       /* Construct graph */
2366       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2367       for (i = 0; i < numVerts; ++i) {
2368         for (j = 0, k = 0; j < numVerts; ++j) {
2369           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2370             graph[i * numVerts + j] = 1;
2371             ++k;
2372           }
2373         }
2374         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid icosahedron, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2375       }
2376       /* Build Topology */
2377       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2378       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2379       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2380       /* Cells */
2381       for (i = 0, c = 0; i < numVerts; ++i) {
2382         for (j = 0; j < i; ++j) {
2383           for (k = 0; k < j; ++k) {
2384             if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i]) {
2385               cone[0] = firstVertex + i;
2386               cone[1] = firstVertex + j;
2387               cone[2] = firstVertex + k;
2388               /* Check orientation */
2389               {
2390                 const PetscInt epsilon[3][3][3] = {
2391                   {{0, 0, 0},  {0, 0, 1},  {0, -1, 0}},
2392                   {{0, 0, -1}, {0, 0, 0},  {1, 0, 0} },
2393                   {{0, 1, 0},  {-1, 0, 0}, {0, 0, 0} }
2394                 };
2395                 PetscReal normal[3];
2396                 PetscInt  e, f;
2397 
2398                 for (d = 0; d < embedDim; ++d) {
2399                   normal[d] = 0.0;
2400                   for (e = 0; e < embedDim; ++e) {
2401                     for (f = 0; f < embedDim; ++f) normal[d] += epsilon[d][e][f] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]);
2402                   }
2403                 }
2404                 if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2405                   PetscInt tmp = cone[1];
2406                   cone[1]      = cone[2];
2407                   cone[2]      = tmp;
2408                 }
2409               }
2410               PetscCall(DMPlexSetCone(dm, c++, cone));
2411             }
2412           }
2413         }
2414       }
2415       PetscCall(DMPlexSymmetrize(dm));
2416       PetscCall(DMPlexStratify(dm));
2417       PetscCall(PetscFree(graph));
2418     } else {
2419       /*
2420         12-21--13
2421          |     |
2422         25  4  24
2423          |     |
2424   12-25--9-16--8-24--13
2425    |     |     |     |
2426   23  5 17  0 15  3  22
2427    |     |     |     |
2428   10-20--6-14--7-19--11
2429          |     |
2430         20  1  19
2431          |     |
2432         10-18--11
2433          |     |
2434         23  2  22
2435          |     |
2436         12-21--13
2437        */
2438       PetscInt cone[4], ornt[4];
2439 
2440       numCells    = rank == 0 ? 6 : 0;
2441       numEdges    = rank == 0 ? 12 : 0;
2442       numVerts    = rank == 0 ? 8 : 0;
2443       firstVertex = numCells;
2444       firstEdge   = numCells + numVerts;
2445       /* Build Topology */
2446       PetscCall(DMPlexSetChart(dm, 0, numCells + numEdges + numVerts));
2447       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 4));
2448       for (e = firstEdge; e < firstEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
2449       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2450       if (rank == 0) {
2451         /* Cell 0 */
2452         cone[0] = 14;
2453         cone[1] = 15;
2454         cone[2] = 16;
2455         cone[3] = 17;
2456         PetscCall(DMPlexSetCone(dm, 0, cone));
2457         ornt[0] = 0;
2458         ornt[1] = 0;
2459         ornt[2] = 0;
2460         ornt[3] = 0;
2461         PetscCall(DMPlexSetConeOrientation(dm, 0, ornt));
2462         /* Cell 1 */
2463         cone[0] = 18;
2464         cone[1] = 19;
2465         cone[2] = 14;
2466         cone[3] = 20;
2467         PetscCall(DMPlexSetCone(dm, 1, cone));
2468         ornt[0] = 0;
2469         ornt[1] = 0;
2470         ornt[2] = -1;
2471         ornt[3] = 0;
2472         PetscCall(DMPlexSetConeOrientation(dm, 1, ornt));
2473         /* Cell 2 */
2474         cone[0] = 21;
2475         cone[1] = 22;
2476         cone[2] = 18;
2477         cone[3] = 23;
2478         PetscCall(DMPlexSetCone(dm, 2, cone));
2479         ornt[0] = 0;
2480         ornt[1] = 0;
2481         ornt[2] = -1;
2482         ornt[3] = 0;
2483         PetscCall(DMPlexSetConeOrientation(dm, 2, ornt));
2484         /* Cell 3 */
2485         cone[0] = 19;
2486         cone[1] = 22;
2487         cone[2] = 24;
2488         cone[3] = 15;
2489         PetscCall(DMPlexSetCone(dm, 3, cone));
2490         ornt[0] = -1;
2491         ornt[1] = -1;
2492         ornt[2] = 0;
2493         ornt[3] = -1;
2494         PetscCall(DMPlexSetConeOrientation(dm, 3, ornt));
2495         /* Cell 4 */
2496         cone[0] = 16;
2497         cone[1] = 24;
2498         cone[2] = 21;
2499         cone[3] = 25;
2500         PetscCall(DMPlexSetCone(dm, 4, cone));
2501         ornt[0] = -1;
2502         ornt[1] = -1;
2503         ornt[2] = -1;
2504         ornt[3] = 0;
2505         PetscCall(DMPlexSetConeOrientation(dm, 4, ornt));
2506         /* Cell 5 */
2507         cone[0] = 20;
2508         cone[1] = 17;
2509         cone[2] = 25;
2510         cone[3] = 23;
2511         PetscCall(DMPlexSetCone(dm, 5, cone));
2512         ornt[0] = -1;
2513         ornt[1] = -1;
2514         ornt[2] = -1;
2515         ornt[3] = -1;
2516         PetscCall(DMPlexSetConeOrientation(dm, 5, ornt));
2517         /* Edges */
2518         cone[0] = 6;
2519         cone[1] = 7;
2520         PetscCall(DMPlexSetCone(dm, 14, cone));
2521         cone[0] = 7;
2522         cone[1] = 8;
2523         PetscCall(DMPlexSetCone(dm, 15, cone));
2524         cone[0] = 8;
2525         cone[1] = 9;
2526         PetscCall(DMPlexSetCone(dm, 16, cone));
2527         cone[0] = 9;
2528         cone[1] = 6;
2529         PetscCall(DMPlexSetCone(dm, 17, cone));
2530         cone[0] = 10;
2531         cone[1] = 11;
2532         PetscCall(DMPlexSetCone(dm, 18, cone));
2533         cone[0] = 11;
2534         cone[1] = 7;
2535         PetscCall(DMPlexSetCone(dm, 19, cone));
2536         cone[0] = 6;
2537         cone[1] = 10;
2538         PetscCall(DMPlexSetCone(dm, 20, cone));
2539         cone[0] = 12;
2540         cone[1] = 13;
2541         PetscCall(DMPlexSetCone(dm, 21, cone));
2542         cone[0] = 13;
2543         cone[1] = 11;
2544         PetscCall(DMPlexSetCone(dm, 22, cone));
2545         cone[0] = 10;
2546         cone[1] = 12;
2547         PetscCall(DMPlexSetCone(dm, 23, cone));
2548         cone[0] = 13;
2549         cone[1] = 8;
2550         PetscCall(DMPlexSetCone(dm, 24, cone));
2551         cone[0] = 12;
2552         cone[1] = 9;
2553         PetscCall(DMPlexSetCone(dm, 25, cone));
2554       }
2555       PetscCall(DMPlexSymmetrize(dm));
2556       PetscCall(DMPlexStratify(dm));
2557       /* Build coordinates */
2558       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2559       if (rank == 0) {
2560         coordsIn[0 * embedDim + 0] = -R;
2561         coordsIn[0 * embedDim + 1] = R;
2562         coordsIn[0 * embedDim + 2] = -R;
2563         coordsIn[1 * embedDim + 0] = R;
2564         coordsIn[1 * embedDim + 1] = R;
2565         coordsIn[1 * embedDim + 2] = -R;
2566         coordsIn[2 * embedDim + 0] = R;
2567         coordsIn[2 * embedDim + 1] = -R;
2568         coordsIn[2 * embedDim + 2] = -R;
2569         coordsIn[3 * embedDim + 0] = -R;
2570         coordsIn[3 * embedDim + 1] = -R;
2571         coordsIn[3 * embedDim + 2] = -R;
2572         coordsIn[4 * embedDim + 0] = -R;
2573         coordsIn[4 * embedDim + 1] = R;
2574         coordsIn[4 * embedDim + 2] = R;
2575         coordsIn[5 * embedDim + 0] = R;
2576         coordsIn[5 * embedDim + 1] = R;
2577         coordsIn[5 * embedDim + 2] = R;
2578         coordsIn[6 * embedDim + 0] = -R;
2579         coordsIn[6 * embedDim + 1] = -R;
2580         coordsIn[6 * embedDim + 2] = R;
2581         coordsIn[7 * embedDim + 0] = R;
2582         coordsIn[7 * embedDim + 1] = -R;
2583         coordsIn[7 * embedDim + 2] = R;
2584       }
2585     }
2586     break;
2587   case 3:
2588     if (simplex) {
2589       const PetscReal edgeLen         = 1.0 / PETSC_PHI;
2590       PetscReal       vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
2591       PetscReal       vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
2592       PetscReal       vertexC[4]      = {0.5, 0.5 * PETSC_PHI, 0.5 / PETSC_PHI, 0.0};
2593       const PetscInt  degree          = 12;
2594       PetscInt        s[4]            = {1, 1, 1};
2595       PetscInt        evenPerm[12][4] = {
2596         {0, 1, 2, 3},
2597         {0, 2, 3, 1},
2598         {0, 3, 1, 2},
2599         {1, 0, 3, 2},
2600         {1, 2, 0, 3},
2601         {1, 3, 2, 0},
2602         {2, 0, 1, 3},
2603         {2, 1, 3, 0},
2604         {2, 3, 0, 1},
2605         {3, 0, 2, 1},
2606         {3, 1, 0, 2},
2607         {3, 2, 1, 0}
2608       };
2609       PetscInt  cone[4];
2610       PetscInt *graph, p, i, j, k, l;
2611 
2612       vertexA[0] *= R;
2613       vertexA[1] *= R;
2614       vertexA[2] *= R;
2615       vertexA[3] *= R;
2616       vertexB[0] *= R;
2617       vertexB[1] *= R;
2618       vertexB[2] *= R;
2619       vertexB[3] *= R;
2620       vertexC[0] *= R;
2621       vertexC[1] *= R;
2622       vertexC[2] *= R;
2623       vertexC[3] *= R;
2624       numCells    = rank == 0 ? 600 : 0;
2625       numVerts    = rank == 0 ? 120 : 0;
2626       firstVertex = numCells;
2627       /* Use the 600-cell, which for a unit sphere has coordinates which are
2628 
2629            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
2630                (\pm 1,    0,       0,      0)  all cyclic permutations   8
2631            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
2632 
2633          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2634          length is then given by 1/\phi = 0.61803.
2635 
2636          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
2637          http://mathworld.wolfram.com/600-Cell.html
2638       */
2639       /* Construct vertices */
2640       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2641       i = 0;
2642       if (rank == 0) {
2643         for (s[0] = -1; s[0] < 2; s[0] += 2) {
2644           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2645             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2646               for (s[3] = -1; s[3] < 2; s[3] += 2) {
2647                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[d] * vertexA[d];
2648                 ++i;
2649               }
2650             }
2651           }
2652         }
2653         for (p = 0; p < embedDim; ++p) {
2654           s[1] = s[2] = s[3] = 1;
2655           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2656             for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertexB[(d + p) % embedDim];
2657             ++i;
2658           }
2659         }
2660         for (p = 0; p < 12; ++p) {
2661           s[3] = 1;
2662           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2663             for (s[1] = -1; s[1] < 2; s[1] += 2) {
2664               for (s[2] = -1; s[2] < 2; s[2] += 2) {
2665                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[evenPerm[p][d]] * vertexC[evenPerm[p][d]];
2666                 ++i;
2667               }
2668             }
2669           }
2670         }
2671       }
2672       PetscCheck(i == numVerts, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertices %" PetscInt_FMT " != %" PetscInt_FMT, i, numVerts);
2673       /* Construct graph */
2674       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2675       for (i = 0; i < numVerts; ++i) {
2676         for (j = 0, k = 0; j < numVerts; ++j) {
2677           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2678             graph[i * numVerts + j] = 1;
2679             ++k;
2680           }
2681         }
2682         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2683       }
2684       /* Build Topology */
2685       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2686       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2687       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2688       /* Cells */
2689       if (rank == 0) {
2690         for (i = 0, c = 0; i < numVerts; ++i) {
2691           for (j = 0; j < i; ++j) {
2692             for (k = 0; k < j; ++k) {
2693               for (l = 0; l < k; ++l) {
2694                 if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i] && graph[l * numVerts + i] && graph[l * numVerts + j] && graph[l * numVerts + k]) {
2695                   cone[0] = firstVertex + i;
2696                   cone[1] = firstVertex + j;
2697                   cone[2] = firstVertex + k;
2698                   cone[3] = firstVertex + l;
2699                   /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
2700                   {
2701                     const PetscInt epsilon[4][4][4][4] = {
2702                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, -1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 0, 0}, {0, 1, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 1, 0}, {0, -1, 0, 0}, {0, 0, 0, 0}}},
2703 
2704                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}}, {{0, 0, -1, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}}},
2705 
2706                       {{{0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, 0, 0}, {0, -1, 0, 0}}, {{0, 0, 0, -1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 1, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}},
2707 
2708                       {{{0, 0, 0, 0}, {0, 0, -1, 0}, {0, 1, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 1, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}}, {{0, -1, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}} }
2709                     };
2710                     PetscReal normal[4];
2711                     PetscInt  e, f, g;
2712 
2713                     for (d = 0; d < embedDim; ++d) {
2714                       normal[d] = 0.0;
2715                       for (e = 0; e < embedDim; ++e) {
2716                         for (f = 0; f < embedDim; ++f) {
2717                           for (g = 0; g < embedDim; ++g) {
2718                             normal[d] += epsilon[d][e][f][g] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]) * (coordsIn[l * embedDim + f] - coordsIn[i * embedDim + f]);
2719                           }
2720                         }
2721                       }
2722                     }
2723                     if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2724                       PetscInt tmp = cone[1];
2725                       cone[1]      = cone[2];
2726                       cone[2]      = tmp;
2727                     }
2728                   }
2729                   PetscCall(DMPlexSetCone(dm, c++, cone));
2730                 }
2731               }
2732             }
2733           }
2734         }
2735       }
2736       PetscCall(DMPlexSymmetrize(dm));
2737       PetscCall(DMPlexStratify(dm));
2738       PetscCall(PetscFree(graph));
2739     }
2740     break;
2741   default:
2742     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unsupported dimension for sphere: %" PetscInt_FMT, dim);
2743   }
2744   /* Create coordinates */
2745   PetscCall(DMGetCoordinateSection(dm, &coordSection));
2746   PetscCall(PetscSectionSetNumFields(coordSection, 1));
2747   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, embedDim));
2748   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVerts));
2749   for (v = firstVertex; v < firstVertex + numVerts; ++v) {
2750     PetscCall(PetscSectionSetDof(coordSection, v, embedDim));
2751     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, embedDim));
2752   }
2753   PetscCall(PetscSectionSetUp(coordSection));
2754   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2755   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2756   PetscCall(VecSetBlockSize(coordinates, embedDim));
2757   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2758   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2759   PetscCall(VecSetType(coordinates, VECSTANDARD));
2760   PetscCall(VecGetArray(coordinates, &coords));
2761   for (v = 0; v < numVerts; ++v)
2762     for (d = 0; d < embedDim; ++d) coords[v * embedDim + d] = coordsIn[v * embedDim + d];
2763   PetscCall(VecRestoreArray(coordinates, &coords));
2764   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2765   PetscCall(VecDestroy(&coordinates));
2766   PetscCall(PetscFree(coordsIn));
2767   {
2768     DM          cdm;
2769     PetscDS     cds;
2770     PetscScalar c = R;
2771 
2772     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToSphere));
2773     PetscCall(DMGetCoordinateDM(dm, &cdm));
2774     PetscCall(DMGetDS(cdm, &cds));
2775     PetscCall(PetscDSSetConstants(cds, 1, &c));
2776   }
2777   /* Wait for coordinate creation before doing in-place modification */
2778   if (simplex) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2779   PetscFunctionReturn(0);
2780 }
2781 
2782 typedef void (*TPSEvaluateFunc)(const PetscReal[], PetscReal *, PetscReal[], PetscReal (*)[3]);
2783 
2784 /*
2785  The Schwarz P implicit surface is
2786 
2787      f(x) = cos(x0) + cos(x1) + cos(x2) = 0
2788 */
2789 static void TPSEvaluate_SchwarzP(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2790 {
2791   PetscReal c[3] = {PetscCosReal(y[0] * PETSC_PI), PetscCosReal(y[1] * PETSC_PI), PetscCosReal(y[2] * PETSC_PI)};
2792   PetscReal g[3] = {-PetscSinReal(y[0] * PETSC_PI), -PetscSinReal(y[1] * PETSC_PI), -PetscSinReal(y[2] * PETSC_PI)};
2793   f[0]           = c[0] + c[1] + c[2];
2794   for (PetscInt i = 0; i < 3; i++) {
2795     grad[i] = PETSC_PI * g[i];
2796     for (PetscInt j = 0; j < 3; j++) hess[i][j] = (i == j) ? -PetscSqr(PETSC_PI) * c[i] : 0.;
2797   }
2798 }
2799 
2800 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2801 static PetscErrorCode TPSExtrudeNormalFunc_SchwarzP(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2802 {
2803   for (PetscInt i = 0; i < 3; i++) u[i] = -PETSC_PI * PetscSinReal(x[i] * PETSC_PI);
2804   return 0;
2805 }
2806 
2807 /*
2808  The Gyroid implicit surface is
2809 
2810  f(x,y,z) = sin(pi * x) * cos (pi * (y + 1/2))  + sin(pi * (y + 1/2)) * cos(pi * (z + 1/4)) + sin(pi * (z + 1/4)) * cos(pi * x)
2811 
2812 */
2813 static void TPSEvaluate_Gyroid(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2814 {
2815   PetscReal s[3] = {PetscSinReal(PETSC_PI * y[0]), PetscSinReal(PETSC_PI * (y[1] + .5)), PetscSinReal(PETSC_PI * (y[2] + .25))};
2816   PetscReal c[3] = {PetscCosReal(PETSC_PI * y[0]), PetscCosReal(PETSC_PI * (y[1] + .5)), PetscCosReal(PETSC_PI * (y[2] + .25))};
2817   f[0]           = s[0] * c[1] + s[1] * c[2] + s[2] * c[0];
2818   grad[0]        = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2819   grad[1]        = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2820   grad[2]        = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2821   hess[0][0]     = -PetscSqr(PETSC_PI) * (s[0] * c[1] + s[2] * c[0]);
2822   hess[0][1]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2823   hess[0][2]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2824   hess[1][0]     = -PetscSqr(PETSC_PI) * (s[1] * c[2] + s[0] * c[1]);
2825   hess[1][1]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2826   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2827   hess[2][0]     = -PetscSqr(PETSC_PI) * (s[2] * c[0] + s[1] * c[2]);
2828   hess[2][1]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2829   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2830 }
2831 
2832 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2833 static PetscErrorCode TPSExtrudeNormalFunc_Gyroid(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2834 {
2835   PetscReal s[3] = {PetscSinReal(PETSC_PI * x[0]), PetscSinReal(PETSC_PI * (x[1] + .5)), PetscSinReal(PETSC_PI * (x[2] + .25))};
2836   PetscReal c[3] = {PetscCosReal(PETSC_PI * x[0]), PetscCosReal(PETSC_PI * (x[1] + .5)), PetscCosReal(PETSC_PI * (x[2] + .25))};
2837   u[0]           = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2838   u[1]           = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2839   u[2]           = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2840   return 0;
2841 }
2842 
2843 /*
2844    We wish to solve
2845 
2846          min_y || y - x ||^2  subject to f(y) = 0
2847 
2848    Let g(y) = grad(f).  The minimization problem is equivalent to asking to satisfy
2849    f(y) = 0 and (y-x) is parallel to g(y).  We do this by using Householder QR to obtain a basis for the
2850    tangent space and ask for both components in the tangent space to be zero.
2851 
2852    Take g to be a column vector and compute the "full QR" factorization Q R = g,
2853    where Q = I - 2 n n^T is a symmetric orthogonal matrix.
2854    The first column of Q is parallel to g so the remaining two columns span the null space.
2855    Let Qn = Q[:,1:] be those remaining columns.  Then Qn Qn^T is an orthogonal projector into the tangent space.
2856    Since Q is symmetric, this is equivalent to multipyling by Q and taking the last two entries.
2857    In total, we have a system of 3 equations in 3 unknowns:
2858 
2859      f(y) = 0                       1 equation
2860      Qn^T (y - x) = 0               2 equations
2861 
2862    Here, we compute the residual and Jacobian of this system.
2863 */
2864 static void TPSNearestPointResJac(TPSEvaluateFunc feval, const PetscScalar x[], const PetscScalar y[], PetscScalar res[], PetscScalar J[])
2865 {
2866   PetscReal yreal[3] = {PetscRealPart(y[0]), PetscRealPart(y[1]), PetscRealPart(y[2])};
2867   PetscReal d[3]     = {PetscRealPart(y[0] - x[0]), PetscRealPart(y[1] - x[1]), PetscRealPart(y[2] - x[2])};
2868   PetscReal f, grad[3], n[3], norm, norm_y[3], nd, nd_y[3], sign;
2869   PetscReal n_y[3][3] = {
2870     {0, 0, 0},
2871     {0, 0, 0},
2872     {0, 0, 0}
2873   };
2874 
2875   feval(yreal, &f, grad, n_y);
2876 
2877   for (PetscInt i = 0; i < 3; i++) n[i] = grad[i];
2878   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2879   for (PetscInt i = 0; i < 3; i++) norm_y[i] = 1. / norm * n[i] * n_y[i][i];
2880 
2881   // Define the Householder reflector
2882   sign = n[0] >= 0 ? 1. : -1.;
2883   n[0] += norm * sign;
2884   for (PetscInt i = 0; i < 3; i++) n_y[0][i] += norm_y[i] * sign;
2885 
2886   norm      = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2887   norm_y[0] = 1. / norm * (n[0] * n_y[0][0]);
2888   norm_y[1] = 1. / norm * (n[0] * n_y[0][1] + n[1] * n_y[1][1]);
2889   norm_y[2] = 1. / norm * (n[0] * n_y[0][2] + n[2] * n_y[2][2]);
2890 
2891   for (PetscInt i = 0; i < 3; i++) {
2892     n[i] /= norm;
2893     for (PetscInt j = 0; j < 3; j++) {
2894       // note that n[i] is n_old[i]/norm when executing the code below
2895       n_y[i][j] = n_y[i][j] / norm - n[i] / norm * norm_y[j];
2896     }
2897   }
2898 
2899   nd = n[0] * d[0] + n[1] * d[1] + n[2] * d[2];
2900   for (PetscInt i = 0; i < 3; i++) nd_y[i] = n[i] + n_y[0][i] * d[0] + n_y[1][i] * d[1] + n_y[2][i] * d[2];
2901 
2902   res[0] = f;
2903   res[1] = d[1] - 2 * n[1] * nd;
2904   res[2] = d[2] - 2 * n[2] * nd;
2905   // J[j][i] is J_{ij} (column major)
2906   for (PetscInt j = 0; j < 3; j++) {
2907     J[0 + j * 3] = grad[j];
2908     J[1 + j * 3] = (j == 1) * 1. - 2 * (n_y[1][j] * nd + n[1] * nd_y[j]);
2909     J[2 + j * 3] = (j == 2) * 1. - 2 * (n_y[2][j] * nd + n[2] * nd_y[j]);
2910   }
2911 }
2912 
2913 /*
2914    Project x to the nearest point on the implicit surface using Newton's method.
2915 */
2916 static PetscErrorCode TPSNearestPoint(TPSEvaluateFunc feval, PetscScalar x[])
2917 {
2918   PetscScalar y[3] = {x[0], x[1], x[2]}; // Initial guess
2919 
2920   PetscFunctionBegin;
2921   for (PetscInt iter = 0; iter < 10; iter++) {
2922     PetscScalar res[3], J[9];
2923     PetscReal   resnorm;
2924     TPSNearestPointResJac(feval, x, y, res, J);
2925     resnorm = PetscSqrtReal(PetscSqr(PetscRealPart(res[0])) + PetscSqr(PetscRealPart(res[1])) + PetscSqr(PetscRealPart(res[2])));
2926     if (0) { // Turn on this monitor if you need to confirm quadratic convergence
2927       PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%" PetscInt_FMT "] res [%g %g %g]\n", iter, (double)PetscRealPart(res[0]), (double)PetscRealPart(res[1]), (double)PetscRealPart(res[2])));
2928     }
2929     if (resnorm < PETSC_SMALL) break;
2930 
2931     // Take the Newton step
2932     PetscCall(PetscKernel_A_gets_inverse_A_3(J, 0., PETSC_FALSE, NULL));
2933     PetscKernel_v_gets_v_minus_A_times_w_3(y, J, res);
2934   }
2935   for (PetscInt i = 0; i < 3; i++) x[i] = y[i];
2936   PetscFunctionReturn(0);
2937 }
2938 
2939 const char *const DMPlexTPSTypes[] = {"SCHWARZ_P", "GYROID", "DMPlexTPSType", "DMPLEX_TPS_", NULL};
2940 
2941 static PetscErrorCode DMPlexCreateTPSMesh_Internal(DM dm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness)
2942 {
2943   PetscMPIInt rank;
2944   PetscInt    topoDim = 2, spaceDim = 3, numFaces = 0, numVertices = 0, numEdges = 0;
2945   PetscInt(*edges)[2] = NULL, *edgeSets = NULL;
2946   PetscInt            *cells_flat = NULL;
2947   PetscReal           *vtxCoords  = NULL;
2948   TPSEvaluateFunc      evalFunc   = NULL;
2949   PetscSimplePointFunc normalFunc = NULL;
2950   DMLabel              label;
2951 
2952   PetscFunctionBegin;
2953   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2954   PetscCheck((layers != 0) ^ (thickness == 0.), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Layers %" PetscInt_FMT " must be nonzero iff thickness %g is nonzero", layers, (double)thickness);
2955   switch (tpstype) {
2956   case DMPLEX_TPS_SCHWARZ_P:
2957     PetscCheck(!periodic || (periodic[0] == DM_BOUNDARY_NONE && periodic[1] == DM_BOUNDARY_NONE && periodic[2] == DM_BOUNDARY_NONE), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Schwarz P does not support periodic meshes");
2958     if (rank == 0) {
2959       PetscInt(*cells)[6][4][4] = NULL; // [junction, junction-face, cell, conn]
2960       PetscInt  Njunctions = 0, Ncuts = 0, Npipes[3], vcount;
2961       PetscReal L = 1;
2962 
2963       Npipes[0]   = (extent[0] + 1) * extent[1] * extent[2];
2964       Npipes[1]   = extent[0] * (extent[1] + 1) * extent[2];
2965       Npipes[2]   = extent[0] * extent[1] * (extent[2] + 1);
2966       Njunctions  = extent[0] * extent[1] * extent[2];
2967       Ncuts       = 2 * (extent[0] * extent[1] + extent[1] * extent[2] + extent[2] * extent[0]);
2968       numVertices = 4 * (Npipes[0] + Npipes[1] + Npipes[2]) + 8 * Njunctions;
2969       PetscCall(PetscMalloc1(3 * numVertices, &vtxCoords));
2970       PetscCall(PetscMalloc1(Njunctions, &cells));
2971       PetscCall(PetscMalloc1(Ncuts * 4, &edges));
2972       PetscCall(PetscMalloc1(Ncuts * 4, &edgeSets));
2973       // x-normal pipes
2974       vcount = 0;
2975       for (PetscInt i = 0; i < extent[0] + 1; i++) {
2976         for (PetscInt j = 0; j < extent[1]; j++) {
2977           for (PetscInt k = 0; k < extent[2]; k++) {
2978             for (PetscInt l = 0; l < 4; l++) {
2979               vtxCoords[vcount++] = (2 * i - 1) * L;
2980               vtxCoords[vcount++] = 2 * j * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2981               vtxCoords[vcount++] = 2 * k * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2982             }
2983           }
2984         }
2985       }
2986       // y-normal pipes
2987       for (PetscInt i = 0; i < extent[0]; i++) {
2988         for (PetscInt j = 0; j < extent[1] + 1; j++) {
2989           for (PetscInt k = 0; k < extent[2]; k++) {
2990             for (PetscInt l = 0; l < 4; l++) {
2991               vtxCoords[vcount++] = 2 * i * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2992               vtxCoords[vcount++] = (2 * j - 1) * L;
2993               vtxCoords[vcount++] = 2 * k * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2994             }
2995           }
2996         }
2997       }
2998       // z-normal pipes
2999       for (PetscInt i = 0; i < extent[0]; i++) {
3000         for (PetscInt j = 0; j < extent[1]; j++) {
3001           for (PetscInt k = 0; k < extent[2] + 1; k++) {
3002             for (PetscInt l = 0; l < 4; l++) {
3003               vtxCoords[vcount++] = 2 * i * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3004               vtxCoords[vcount++] = 2 * j * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3005               vtxCoords[vcount++] = (2 * k - 1) * L;
3006             }
3007           }
3008         }
3009       }
3010       // junctions
3011       for (PetscInt i = 0; i < extent[0]; i++) {
3012         for (PetscInt j = 0; j < extent[1]; j++) {
3013           for (PetscInt k = 0; k < extent[2]; k++) {
3014             const PetscInt J = (i * extent[1] + j) * extent[2] + k, Jvoff = (Npipes[0] + Npipes[1] + Npipes[2]) * 4 + J * 8;
3015             PetscCheck(vcount / 3 == Jvoff, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected vertex count");
3016             for (PetscInt ii = 0; ii < 2; ii++) {
3017               for (PetscInt jj = 0; jj < 2; jj++) {
3018                 for (PetscInt kk = 0; kk < 2; kk++) {
3019                   double Ls           = (1 - sqrt(2) / 4) * L;
3020                   vtxCoords[vcount++] = 2 * i * L + (2 * ii - 1) * Ls;
3021                   vtxCoords[vcount++] = 2 * j * L + (2 * jj - 1) * Ls;
3022                   vtxCoords[vcount++] = 2 * k * L + (2 * kk - 1) * Ls;
3023                 }
3024               }
3025             }
3026             const PetscInt jfaces[3][2][4] = {
3027               {{3, 1, 0, 2}, {7, 5, 4, 6}}, // x-aligned
3028               {{5, 4, 0, 1}, {7, 6, 2, 3}}, // y-aligned
3029               {{6, 2, 0, 4}, {7, 3, 1, 5}}  // z-aligned
3030             };
3031             const PetscInt pipe_lo[3] = {// vertex numbers of pipes
3032                                          ((i * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + Npipes[0] + Npipes[1]) * 4};
3033             const PetscInt pipe_hi[3] = {// vertex numbers of pipes
3034                                          (((i + 1) * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j + 1) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + 1 + Npipes[0] + Npipes[1]) * 4};
3035             for (PetscInt dir = 0; dir < 3; dir++) { // x,y,z
3036               const PetscInt ijk[3] = {i, j, k};
3037               for (PetscInt l = 0; l < 4; l++) { // rotations
3038                 cells[J][dir * 2 + 0][l][0] = pipe_lo[dir] + l;
3039                 cells[J][dir * 2 + 0][l][1] = Jvoff + jfaces[dir][0][l];
3040                 cells[J][dir * 2 + 0][l][2] = Jvoff + jfaces[dir][0][(l - 1 + 4) % 4];
3041                 cells[J][dir * 2 + 0][l][3] = pipe_lo[dir] + (l - 1 + 4) % 4;
3042                 cells[J][dir * 2 + 1][l][0] = Jvoff + jfaces[dir][1][l];
3043                 cells[J][dir * 2 + 1][l][1] = pipe_hi[dir] + l;
3044                 cells[J][dir * 2 + 1][l][2] = pipe_hi[dir] + (l - 1 + 4) % 4;
3045                 cells[J][dir * 2 + 1][l][3] = Jvoff + jfaces[dir][1][(l - 1 + 4) % 4];
3046                 if (ijk[dir] == 0) {
3047                   edges[numEdges][0] = pipe_lo[dir] + l;
3048                   edges[numEdges][1] = pipe_lo[dir] + (l + 1) % 4;
3049                   edgeSets[numEdges] = dir * 2 + 1;
3050                   numEdges++;
3051                 }
3052                 if (ijk[dir] + 1 == extent[dir]) {
3053                   edges[numEdges][0] = pipe_hi[dir] + l;
3054                   edges[numEdges][1] = pipe_hi[dir] + (l + 1) % 4;
3055                   edgeSets[numEdges] = dir * 2 + 2;
3056                   numEdges++;
3057                 }
3058               }
3059             }
3060           }
3061         }
3062       }
3063       PetscCheck(numEdges == Ncuts * 4, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Edge count %" PetscInt_FMT " incompatible with number of cuts %" PetscInt_FMT, numEdges, Ncuts);
3064       numFaces   = 24 * Njunctions;
3065       cells_flat = cells[0][0][0];
3066     }
3067     evalFunc   = TPSEvaluate_SchwarzP;
3068     normalFunc = TPSExtrudeNormalFunc_SchwarzP;
3069     break;
3070   case DMPLEX_TPS_GYROID:
3071     if (rank == 0) {
3072       // This is a coarse mesh approximation of the gyroid shifted to being the zero of the level set
3073       //
3074       //     sin(pi*x)*cos(pi*(y+1/2)) + sin(pi*(y+1/2))*cos(pi*(z+1/4)) + sin(pi*(z+1/4))*cos(x)
3075       //
3076       // on the cell [0,2]^3.
3077       //
3078       // Think about dividing that cell into four columns, and focus on the column [0,1]x[0,1]x[0,2].
3079       // If you looked at the gyroid in that column at different slices of z you would see that it kind of spins
3080       // like a boomerang:
3081       //
3082       //     z = 0          z = 1/4        z = 1/2        z = 3/4     //
3083       //     -----          -------        -------        -------     //
3084       //                                                              //
3085       //     +       +      +       +      +       +      +   \   +   //
3086       //      \                                   /            \      //
3087       //       \            `-_   _-'            /              }     //
3088       //        *-_            `-'            _-'              /      //
3089       //     +     `-+      +       +      +-'     +      +   /   +   //
3090       //                                                              //
3091       //                                                              //
3092       //     z = 1          z = 5/4        z = 3/2        z = 7/4     //
3093       //     -----          -------        -------        -------     //
3094       //                                                              //
3095       //     +-_     +      +       +      +     _-+      +   /   +   //
3096       //        `-_            _-_            _-`            /        //
3097       //           \        _-'   `-_        /              {         //
3098       //            \                       /                \        //
3099       //     +       +      +       +      +       +      +   \   +   //
3100       //
3101       //
3102       // This course mesh approximates each of these slices by two line segments,
3103       // and then connects the segments in consecutive layers with quadrilateral faces.
3104       // All of the end points of the segments are multiples of 1/4 except for the
3105       // point * in the picture for z = 0 above and the similar points in other layers.
3106       // That point is at (gamma, gamma, 0), where gamma is calculated below.
3107       //
3108       // The column  [1,2]x[1,2]x[0,2] looks the same as this column;
3109       // The columns [1,2]x[0,1]x[0,2] and [0,1]x[1,2]x[0,2] are mirror images.
3110       //
3111       // As for how this method turned into the names given to the vertices:
3112       // that was not systematic, it was just the way it worked out in my handwritten notes.
3113 
3114       PetscInt facesPerBlock = 64;
3115       PetscInt vertsPerBlock = 56;
3116       PetscInt extentPlus[3];
3117       PetscInt numBlocks, numBlocksPlus;
3118       const PetscInt A = 0, B = 1, C = 2, D = 3, E = 4, F = 5, G = 6, H = 7, II = 8, J = 9, K = 10, L = 11, M = 12, N = 13, O = 14, P = 15, Q = 16, R = 17, S = 18, T = 19, U = 20, V = 21, W = 22, X = 23, Y = 24, Z = 25, Ap = 26, Bp = 27, Cp = 28, Dp = 29, Ep = 30, Fp = 31, Gp = 32, Hp = 33, Ip = 34, Jp = 35, Kp = 36, Lp = 37, Mp = 38, Np = 39, Op = 40, Pp = 41, Qp = 42, Rp = 43, Sp = 44, Tp = 45, Up = 46, Vp = 47, Wp = 48, Xp = 49, Yp = 50, Zp = 51, Aq = 52, Bq = 53, Cq = 54, Dq = 55;
3119       const PetscInt pattern[64][4] = {
3120   /* face to vertex within the coarse discretization of a single gyroid block */
3121   /* layer 0 */
3122         {A,           C,           K,           G          },
3123         {C,           B,           II,          K          },
3124         {D,           A,           H,           L          },
3125         {B + 56 * 1,  D,           L,           J          },
3126         {E,           B + 56 * 1,  J,           N          },
3127         {A + 56 * 2,  E,           N,           H + 56 * 2 },
3128         {F,           A + 56 * 2,  G + 56 * 2,  M          },
3129         {B,           F,           M,           II         },
3130  /* layer 1 */
3131         {G,           K,           Q,           O          },
3132         {K,           II,          P,           Q          },
3133         {L,           H,           O + 56 * 1,  R          },
3134         {J,           L,           R,           P          },
3135         {N,           J,           P,           S          },
3136         {H + 56 * 2,  N,           S,           O + 56 * 3 },
3137         {M,           G + 56 * 2,  O + 56 * 2,  T          },
3138         {II,          M,           T,           P          },
3139  /* layer 2 */
3140         {O,           Q,           Y,           U          },
3141         {Q,           P,           W,           Y          },
3142         {R,           O + 56 * 1,  U + 56 * 1,  Ap         },
3143         {P,           R,           Ap,          W          },
3144         {S,           P,           X,           Bp         },
3145         {O + 56 * 3,  S,           Bp,          V + 56 * 1 },
3146         {T,           O + 56 * 2,  V,           Z          },
3147         {P,           T,           Z,           X          },
3148  /* layer 3 */
3149         {U,           Y,           Ep,          Dp         },
3150         {Y,           W,           Cp,          Ep         },
3151         {Ap,          U + 56 * 1,  Dp + 56 * 1, Gp         },
3152         {W,           Ap,          Gp,          Cp         },
3153         {Bp,          X,           Cp + 56 * 2, Fp         },
3154         {V + 56 * 1,  Bp,          Fp,          Dp + 56 * 1},
3155         {Z,           V,           Dp,          Hp         },
3156         {X,           Z,           Hp,          Cp + 56 * 2},
3157  /* layer 4 */
3158         {Dp,          Ep,          Mp,          Kp         },
3159         {Ep,          Cp,          Ip,          Mp         },
3160         {Gp,          Dp + 56 * 1, Lp,          Np         },
3161         {Cp,          Gp,          Np,          Jp         },
3162         {Fp,          Cp + 56 * 2, Jp + 56 * 2, Pp         },
3163         {Dp + 56 * 1, Fp,          Pp,          Lp         },
3164         {Hp,          Dp,          Kp,          Op         },
3165         {Cp + 56 * 2, Hp,          Op,          Ip + 56 * 2},
3166  /* layer 5 */
3167         {Kp,          Mp,          Sp,          Rp         },
3168         {Mp,          Ip,          Qp,          Sp         },
3169         {Np,          Lp,          Rp,          Tp         },
3170         {Jp,          Np,          Tp,          Qp + 56 * 1},
3171         {Pp,          Jp + 56 * 2, Qp + 56 * 3, Up         },
3172         {Lp,          Pp,          Up,          Rp         },
3173         {Op,          Kp,          Rp,          Vp         },
3174         {Ip + 56 * 2, Op,          Vp,          Qp + 56 * 2},
3175  /* layer 6 */
3176         {Rp,          Sp,          Aq,          Yp         },
3177         {Sp,          Qp,          Wp,          Aq         },
3178         {Tp,          Rp,          Yp,          Cq         },
3179         {Qp + 56 * 1, Tp,          Cq,          Wp + 56 * 1},
3180         {Up,          Qp + 56 * 3, Xp + 56 * 1, Dq         },
3181         {Rp,          Up,          Dq,          Zp         },
3182         {Vp,          Rp,          Zp,          Bq         },
3183         {Qp + 56 * 2, Vp,          Bq,          Xp         },
3184  /* layer 7 (the top is the periodic image of the bottom of layer 0) */
3185         {Yp,          Aq,          C + 56 * 4,  A + 56 * 4 },
3186         {Aq,          Wp,          B + 56 * 4,  C + 56 * 4 },
3187         {Cq,          Yp,          A + 56 * 4,  D + 56 * 4 },
3188         {Wp + 56 * 1, Cq,          D + 56 * 4,  B + 56 * 5 },
3189         {Dq,          Xp + 56 * 1, B + 56 * 5,  E + 56 * 4 },
3190         {Zp,          Dq,          E + 56 * 4,  A + 56 * 6 },
3191         {Bq,          Zp,          A + 56 * 6,  F + 56 * 4 },
3192         {Xp,          Bq,          F + 56 * 4,  B + 56 * 4 }
3193       };
3194       const PetscReal gamma                = PetscAcosReal((PetscSqrtReal(3.) - 1.) / PetscSqrtReal(2.)) / PETSC_PI;
3195       const PetscReal patternCoords[56][3] = {
3196         {1.,        0.,        0.  }, /* A  */
3197         {0.,        1.,        0.  }, /* B  */
3198         {gamma,     gamma,     0.  }, /* C  */
3199         {1 + gamma, 1 - gamma, 0.  }, /* D  */
3200         {2 - gamma, 2 - gamma, 0.  }, /* E  */
3201         {1 - gamma, 1 + gamma, 0.  }, /* F  */
3202 
3203         {.5,        0,         .25 }, /* G  */
3204         {1.5,       0.,        .25 }, /* H  */
3205         {.5,        1.,        .25 }, /* II */
3206         {1.5,       1.,        .25 }, /* J  */
3207         {.25,       .5,        .25 }, /* K  */
3208         {1.25,      .5,        .25 }, /* L  */
3209         {.75,       1.5,       .25 }, /* M  */
3210         {1.75,      1.5,       .25 }, /* N  */
3211 
3212         {0.,        0.,        .5  }, /* O  */
3213         {1.,        1.,        .5  }, /* P  */
3214         {gamma,     1 - gamma, .5  }, /* Q  */
3215         {1 + gamma, gamma,     .5  }, /* R  */
3216         {2 - gamma, 1 + gamma, .5  }, /* S  */
3217         {1 - gamma, 2 - gamma, .5  }, /* T  */
3218 
3219         {0.,        .5,        .75 }, /* U  */
3220         {0.,        1.5,       .75 }, /* V  */
3221         {1.,        .5,        .75 }, /* W  */
3222         {1.,        1.5,       .75 }, /* X  */
3223         {.5,        .75,       .75 }, /* Y  */
3224         {.5,        1.75,      .75 }, /* Z  */
3225         {1.5,       .25,       .75 }, /* Ap */
3226         {1.5,       1.25,      .75 }, /* Bp */
3227 
3228         {1.,        0.,        1.  }, /* Cp */
3229         {0.,        1.,        1.  }, /* Dp */
3230         {1 - gamma, 1 - gamma, 1.  }, /* Ep */
3231         {1 + gamma, 1 + gamma, 1.  }, /* Fp */
3232         {2 - gamma, gamma,     1.  }, /* Gp */
3233         {gamma,     2 - gamma, 1.  }, /* Hp */
3234 
3235         {.5,        0.,        1.25}, /* Ip */
3236         {1.5,       0.,        1.25}, /* Jp */
3237         {.5,        1.,        1.25}, /* Kp */
3238         {1.5,       1.,        1.25}, /* Lp */
3239         {.75,       .5,        1.25}, /* Mp */
3240         {1.75,      .5,        1.25}, /* Np */
3241         {.25,       1.5,       1.25}, /* Op */
3242         {1.25,      1.5,       1.25}, /* Pp */
3243 
3244         {0.,        0.,        1.5 }, /* Qp */
3245         {1.,        1.,        1.5 }, /* Rp */
3246         {1 - gamma, gamma,     1.5 }, /* Sp */
3247         {2 - gamma, 1 - gamma, 1.5 }, /* Tp */
3248         {1 + gamma, 2 - gamma, 1.5 }, /* Up */
3249         {gamma,     1 + gamma, 1.5 }, /* Vp */
3250 
3251         {0.,        .5,        1.75}, /* Wp */
3252         {0.,        1.5,       1.75}, /* Xp */
3253         {1.,        .5,        1.75}, /* Yp */
3254         {1.,        1.5,       1.75}, /* Zp */
3255         {.5,        .25,       1.75}, /* Aq */
3256         {.5,        1.25,      1.75}, /* Bq */
3257         {1.5,       .75,       1.75}, /* Cq */
3258         {1.5,       1.75,      1.75}, /* Dq */
3259       };
3260       PetscInt(*cells)[64][4] = NULL;
3261       PetscBool *seen;
3262       PetscInt  *vertToTrueVert;
3263       PetscInt   count;
3264 
3265       for (PetscInt i = 0; i < 3; i++) extentPlus[i] = extent[i] + 1;
3266       numBlocks = 1;
3267       for (PetscInt i = 0; i < 3; i++) numBlocks *= extent[i];
3268       numBlocksPlus = 1;
3269       for (PetscInt i = 0; i < 3; i++) numBlocksPlus *= extentPlus[i];
3270       numFaces = numBlocks * facesPerBlock;
3271       PetscCall(PetscMalloc1(numBlocks, &cells));
3272       PetscCall(PetscCalloc1(numBlocksPlus * vertsPerBlock, &seen));
3273       for (PetscInt k = 0; k < extent[2]; k++) {
3274         for (PetscInt j = 0; j < extent[1]; j++) {
3275           for (PetscInt i = 0; i < extent[0]; i++) {
3276             for (PetscInt f = 0; f < facesPerBlock; f++) {
3277               for (PetscInt v = 0; v < 4; v++) {
3278                 PetscInt vertRaw     = pattern[f][v];
3279                 PetscInt blockidx    = vertRaw / 56;
3280                 PetscInt patternvert = vertRaw % 56;
3281                 PetscInt xplus       = (blockidx & 1);
3282                 PetscInt yplus       = (blockidx & 2) >> 1;
3283                 PetscInt zplus       = (blockidx & 4) >> 2;
3284                 PetscInt zcoord      = (periodic && periodic[2] == DM_BOUNDARY_PERIODIC) ? ((k + zplus) % extent[2]) : (k + zplus);
3285                 PetscInt ycoord      = (periodic && periodic[1] == DM_BOUNDARY_PERIODIC) ? ((j + yplus) % extent[1]) : (j + yplus);
3286                 PetscInt xcoord      = (periodic && periodic[0] == DM_BOUNDARY_PERIODIC) ? ((i + xplus) % extent[0]) : (i + xplus);
3287                 PetscInt vert        = ((zcoord * extentPlus[1] + ycoord) * extentPlus[0] + xcoord) * 56 + patternvert;
3288 
3289                 cells[(k * extent[1] + j) * extent[0] + i][f][v] = vert;
3290                 seen[vert]                                       = PETSC_TRUE;
3291               }
3292             }
3293           }
3294         }
3295       }
3296       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++)
3297         if (seen[i]) numVertices++;
3298       count = 0;
3299       PetscCall(PetscMalloc1(numBlocksPlus * vertsPerBlock, &vertToTrueVert));
3300       PetscCall(PetscMalloc1(numVertices * 3, &vtxCoords));
3301       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) vertToTrueVert[i] = -1;
3302       for (PetscInt k = 0; k < extentPlus[2]; k++) {
3303         for (PetscInt j = 0; j < extentPlus[1]; j++) {
3304           for (PetscInt i = 0; i < extentPlus[0]; i++) {
3305             for (PetscInt v = 0; v < vertsPerBlock; v++) {
3306               PetscInt vIdx = ((k * extentPlus[1] + j) * extentPlus[0] + i) * vertsPerBlock + v;
3307 
3308               if (seen[vIdx]) {
3309                 PetscInt thisVert;
3310 
3311                 vertToTrueVert[vIdx] = thisVert = count++;
3312 
3313                 for (PetscInt d = 0; d < 3; d++) vtxCoords[3 * thisVert + d] = patternCoords[v][d];
3314                 vtxCoords[3 * thisVert + 0] += i * 2;
3315                 vtxCoords[3 * thisVert + 1] += j * 2;
3316                 vtxCoords[3 * thisVert + 2] += k * 2;
3317               }
3318             }
3319           }
3320         }
3321       }
3322       for (PetscInt i = 0; i < numBlocks; i++) {
3323         for (PetscInt f = 0; f < facesPerBlock; f++) {
3324           for (PetscInt v = 0; v < 4; v++) cells[i][f][v] = vertToTrueVert[cells[i][f][v]];
3325         }
3326       }
3327       PetscCall(PetscFree(vertToTrueVert));
3328       PetscCall(PetscFree(seen));
3329       cells_flat = cells[0][0];
3330       numEdges   = 0;
3331       for (PetscInt i = 0; i < numFaces; i++) {
3332         for (PetscInt e = 0; e < 4; e++) {
3333           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3334           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3335 
3336           for (PetscInt d = 0; d < 3; d++) {
3337             if (!periodic || periodic[0] != DM_BOUNDARY_PERIODIC) {
3338               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) numEdges++;
3339               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) numEdges++;
3340             }
3341           }
3342         }
3343       }
3344       PetscCall(PetscMalloc1(numEdges, &edges));
3345       PetscCall(PetscMalloc1(numEdges, &edgeSets));
3346       for (PetscInt edge = 0, i = 0; i < numFaces; i++) {
3347         for (PetscInt e = 0; e < 4; e++) {
3348           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3349           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3350 
3351           for (PetscInt d = 0; d < 3; d++) {
3352             if (!periodic || periodic[d] != DM_BOUNDARY_PERIODIC) {
3353               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) {
3354                 edges[edge][0]   = ev[0];
3355                 edges[edge][1]   = ev[1];
3356                 edgeSets[edge++] = 2 * d;
3357               }
3358               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) {
3359                 edges[edge][0]   = ev[0];
3360                 edges[edge][1]   = ev[1];
3361                 edgeSets[edge++] = 2 * d + 1;
3362               }
3363             }
3364           }
3365         }
3366       }
3367     }
3368     evalFunc   = TPSEvaluate_Gyroid;
3369     normalFunc = TPSExtrudeNormalFunc_Gyroid;
3370     break;
3371   }
3372 
3373   PetscCall(DMSetDimension(dm, topoDim));
3374   if (rank == 0) PetscCall(DMPlexBuildFromCellList(dm, numFaces, numVertices, 4, cells_flat));
3375   else PetscCall(DMPlexBuildFromCellList(dm, 0, 0, 0, NULL));
3376   PetscCall(PetscFree(cells_flat));
3377   {
3378     DM idm;
3379     PetscCall(DMPlexInterpolate(dm, &idm));
3380     PetscCall(DMPlexReplace_Internal(dm, &idm));
3381   }
3382   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, vtxCoords));
3383   else PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, NULL));
3384   PetscCall(PetscFree(vtxCoords));
3385 
3386   PetscCall(DMCreateLabel(dm, "Face Sets"));
3387   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3388   for (PetscInt e = 0; e < numEdges; e++) {
3389     PetscInt        njoin;
3390     const PetscInt *join, verts[] = {numFaces + edges[e][0], numFaces + edges[e][1]};
3391     PetscCall(DMPlexGetJoin(dm, 2, verts, &njoin, &join));
3392     PetscCheck(njoin == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected unique join of vertices %" PetscInt_FMT " and %" PetscInt_FMT, edges[e][0], edges[e][1]);
3393     PetscCall(DMLabelSetValue(label, join[0], edgeSets[e]));
3394     PetscCall(DMPlexRestoreJoin(dm, 2, verts, &njoin, &join));
3395   }
3396   PetscCall(PetscFree(edges));
3397   PetscCall(PetscFree(edgeSets));
3398   if (tps_distribute) {
3399     DM               pdm = NULL;
3400     PetscPartitioner part;
3401 
3402     PetscCall(DMPlexGetPartitioner(dm, &part));
3403     PetscCall(PetscPartitionerSetFromOptions(part));
3404     PetscCall(DMPlexDistribute(dm, 0, NULL, &pdm));
3405     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
3406     // Do not auto-distribute again
3407     PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
3408   }
3409 
3410   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3411   for (PetscInt refine = 0; refine < refinements; refine++) {
3412     PetscInt     m;
3413     DM           dmf;
3414     Vec          X;
3415     PetscScalar *x;
3416     PetscCall(DMRefine(dm, MPI_COMM_NULL, &dmf));
3417     PetscCall(DMPlexReplace_Internal(dm, &dmf));
3418 
3419     PetscCall(DMGetCoordinatesLocal(dm, &X));
3420     PetscCall(VecGetLocalSize(X, &m));
3421     PetscCall(VecGetArray(X, &x));
3422     for (PetscInt i = 0; i < m; i += 3) PetscCall(TPSNearestPoint(evalFunc, &x[i]));
3423     PetscCall(VecRestoreArray(X, &x));
3424   }
3425 
3426   // Face Sets has already been propagated to new vertices during refinement; this propagates to the initial vertices.
3427   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3428   PetscCall(DMPlexLabelComplete(dm, label));
3429 
3430   if (thickness > 0) {
3431     DM              edm, cdm, ecdm;
3432     DMPlexTransform tr;
3433     const char     *prefix;
3434     PetscOptions    options;
3435     // Code from DMPlexExtrude
3436     PetscCall(DMPlexTransformCreate(PetscObjectComm((PetscObject)dm), &tr));
3437     PetscCall(DMPlexTransformSetDM(tr, dm));
3438     PetscCall(DMPlexTransformSetType(tr, DMPLEXEXTRUDE));
3439     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
3440     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)tr, prefix));
3441     PetscCall(PetscObjectGetOptions((PetscObject)dm, &options));
3442     PetscCall(PetscObjectSetOptions((PetscObject)tr, options));
3443     PetscCall(DMPlexTransformExtrudeSetLayers(tr, layers));
3444     PetscCall(DMPlexTransformExtrudeSetThickness(tr, thickness));
3445     PetscCall(DMPlexTransformExtrudeSetTensor(tr, PETSC_FALSE));
3446     PetscCall(DMPlexTransformExtrudeSetSymmetric(tr, PETSC_TRUE));
3447     PetscCall(DMPlexTransformExtrudeSetNormalFunction(tr, normalFunc));
3448     PetscCall(DMPlexTransformSetFromOptions(tr));
3449     PetscCall(PetscObjectSetOptions((PetscObject)tr, NULL));
3450     PetscCall(DMPlexTransformSetUp(tr));
3451     PetscCall(PetscObjectViewFromOptions((PetscObject)tr, NULL, "-dm_plex_tps_transform_view"));
3452     PetscCall(DMPlexTransformApply(tr, dm, &edm));
3453     PetscCall(DMCopyDisc(dm, edm));
3454     PetscCall(DMGetCoordinateDM(dm, &cdm));
3455     PetscCall(DMGetCoordinateDM(edm, &ecdm));
3456     PetscCall(DMCopyDisc(cdm, ecdm));
3457     PetscCall(DMPlexTransformCreateDiscLabels(tr, edm));
3458     PetscCall(DMPlexTransformDestroy(&tr));
3459     if (edm) {
3460       ((DM_Plex *)edm->data)->printFEM    = ((DM_Plex *)dm->data)->printFEM;
3461       ((DM_Plex *)edm->data)->printL2     = ((DM_Plex *)dm->data)->printL2;
3462       ((DM_Plex *)edm->data)->printLocate = ((DM_Plex *)dm->data)->printLocate;
3463     }
3464     PetscCall(DMPlexReplace_Internal(dm, &edm));
3465   }
3466   PetscFunctionReturn(0);
3467 }
3468 
3469 /*@
3470   DMPlexCreateTPSMesh - Create a distributed, interpolated mesh of a triply-periodic surface
3471 
3472   Collective
3473 
3474   Input Parameters:
3475 + comm   - The communicator for the `DM` object
3476 . tpstype - Type of triply-periodic surface
3477 . extent - Array of length 3 containing number of periods in each direction
3478 . periodic - array of length 3 with periodicity, or NULL for non-periodic
3479 . tps_distribute - Distribute 2D manifold mesh prior to refinement and extrusion (more scalable)
3480 . refinements - Number of factor-of-2 refinements of 2D manifold mesh
3481 . layers - Number of cell layers extruded in normal direction
3482 - thickness - Thickness in normal direction
3483 
3484   Output Parameter:
3485 . dm  - The `DM` object
3486 
3487   Level: beginner
3488 
3489   Notes:
3490   This meshes the surface of the Schwarz P or Gyroid surfaces.  Schwarz P is is the simplest member of the triply-periodic minimal surfaces.
3491   https://en.wikipedia.org/wiki/Schwarz_minimal_surface#Schwarz_P_(%22Primitive%22) and can be cut with "clean" boundaries.
3492   The Gyroid (https://en.wikipedia.org/wiki/Gyroid) is another triply-periodic minimal surface with applications in additive manufacturing; it is much more difficult to "cut" since there are no planes of symmetry.
3493   Our implementation creates a very coarse mesh of the surface and refines (by 4-way splitting) as many times as requested.
3494   On each refinement, all vertices are projected to their nearest point on the surface.
3495   This projection could readily be extended to related surfaces.
3496 
3497   The face (edge) sets for the Schwarz P surface are numbered 1(-x), 2(+x), 3(-y), 4(+y), 5(-z), 6(+z).
3498   When the mesh is refined, "Face Sets" contain the new vertices (created during refinement).  Use DMPlexLabelComplete() to propagate to coarse-level vertices.
3499 
3500   Developer Note:
3501   The Gyroid mesh does not currently mark boundary sets.
3502 
3503   References:
3504 . * - Maskery et al, Insights into the mechanical properties of several triply periodic minimal surface lattice structures made by polymer additive manufacturing, 2017.
3505     https://doi.org/10.1016/j.polymer.2017.11.049
3506 
3507 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMSetType()`, `DMCreate()`
3508 @*/
3509 PetscErrorCode DMPlexCreateTPSMesh(MPI_Comm comm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness, DM *dm)
3510 {
3511   PetscFunctionBegin;
3512   PetscCall(DMCreate(comm, dm));
3513   PetscCall(DMSetType(*dm, DMPLEX));
3514   PetscCall(DMPlexCreateTPSMesh_Internal(*dm, tpstype, extent, periodic, tps_distribute, refinements, layers, thickness));
3515   PetscFunctionReturn(0);
3516 }
3517 
3518 /*@
3519   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
3520 
3521   Collective
3522 
3523   Input Parameters:
3524 + comm    - The communicator for the `DM` object
3525 . dim     - The dimension
3526 . simplex - Use simplices, or tensor product cells
3527 - R       - The radius
3528 
3529   Output Parameter:
3530 . dm  - The `DM` object
3531 
3532   Level: beginner
3533 
3534 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBallMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3535 @*/
3536 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscReal R, DM *dm)
3537 {
3538   PetscFunctionBegin;
3539   PetscValidPointer(dm, 5);
3540   PetscCall(DMCreate(comm, dm));
3541   PetscCall(DMSetType(*dm, DMPLEX));
3542   PetscCall(DMPlexCreateSphereMesh_Internal(*dm, dim, simplex, R));
3543   PetscFunctionReturn(0);
3544 }
3545 
3546 static PetscErrorCode DMPlexCreateBallMesh_Internal(DM dm, PetscInt dim, PetscReal R)
3547 {
3548   DM      sdm, vol;
3549   DMLabel bdlabel;
3550 
3551   PetscFunctionBegin;
3552   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
3553   PetscCall(DMSetType(sdm, DMPLEX));
3554   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sdm, "bd_"));
3555   PetscCall(DMPlexCreateSphereMesh_Internal(sdm, dim - 1, PETSC_TRUE, R));
3556   PetscCall(DMSetFromOptions(sdm));
3557   PetscCall(DMViewFromOptions(sdm, NULL, "-dm_view"));
3558   PetscCall(DMPlexGenerate(sdm, NULL, PETSC_TRUE, &vol));
3559   PetscCall(DMDestroy(&sdm));
3560   PetscCall(DMPlexReplace_Internal(dm, &vol));
3561   PetscCall(DMCreateLabel(dm, "marker"));
3562   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
3563   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
3564   PetscCall(DMPlexLabelComplete(dm, bdlabel));
3565   PetscFunctionReturn(0);
3566 }
3567 
3568 /*@
3569   DMPlexCreateBallMesh - Creates a simplex mesh on the d-dimensional ball, B^d.
3570 
3571   Collective
3572 
3573   Input Parameters:
3574 + comm  - The communicator for the `DM` object
3575 . dim   - The dimension
3576 - R     - The radius
3577 
3578   Output Parameter:
3579 . dm  - The `DM` object
3580 
3581   Options Database Key:
3582 - bd_dm_refine - This will refine the surface mesh preserving the sphere geometry
3583 
3584   Level: beginner
3585 
3586 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3587 @*/
3588 PetscErrorCode DMPlexCreateBallMesh(MPI_Comm comm, PetscInt dim, PetscReal R, DM *dm)
3589 {
3590   PetscFunctionBegin;
3591   PetscCall(DMCreate(comm, dm));
3592   PetscCall(DMSetType(*dm, DMPLEX));
3593   PetscCall(DMPlexCreateBallMesh_Internal(*dm, dim, R));
3594   PetscFunctionReturn(0);
3595 }
3596 
3597 static PetscErrorCode DMPlexCreateReferenceCell_Internal(DM rdm, DMPolytopeType ct)
3598 {
3599   PetscFunctionBegin;
3600   switch (ct) {
3601   case DM_POLYTOPE_POINT: {
3602     PetscInt    numPoints[1]        = {1};
3603     PetscInt    coneSize[1]         = {0};
3604     PetscInt    cones[1]            = {0};
3605     PetscInt    coneOrientations[1] = {0};
3606     PetscScalar vertexCoords[1]     = {0.0};
3607 
3608     PetscCall(DMSetDimension(rdm, 0));
3609     PetscCall(DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3610   } break;
3611   case DM_POLYTOPE_SEGMENT: {
3612     PetscInt    numPoints[2]        = {2, 1};
3613     PetscInt    coneSize[3]         = {2, 0, 0};
3614     PetscInt    cones[2]            = {1, 2};
3615     PetscInt    coneOrientations[2] = {0, 0};
3616     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3617 
3618     PetscCall(DMSetDimension(rdm, 1));
3619     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3620   } break;
3621   case DM_POLYTOPE_POINT_PRISM_TENSOR: {
3622     PetscInt    numPoints[2]        = {2, 1};
3623     PetscInt    coneSize[3]         = {2, 0, 0};
3624     PetscInt    cones[2]            = {1, 2};
3625     PetscInt    coneOrientations[2] = {0, 0};
3626     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3627 
3628     PetscCall(DMSetDimension(rdm, 1));
3629     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3630   } break;
3631   case DM_POLYTOPE_TRIANGLE: {
3632     PetscInt    numPoints[2]        = {3, 1};
3633     PetscInt    coneSize[4]         = {3, 0, 0, 0};
3634     PetscInt    cones[3]            = {1, 2, 3};
3635     PetscInt    coneOrientations[3] = {0, 0, 0};
3636     PetscScalar vertexCoords[6]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0};
3637 
3638     PetscCall(DMSetDimension(rdm, 2));
3639     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3640   } break;
3641   case DM_POLYTOPE_QUADRILATERAL: {
3642     PetscInt    numPoints[2]        = {4, 1};
3643     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3644     PetscInt    cones[4]            = {1, 2, 3, 4};
3645     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3646     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0};
3647 
3648     PetscCall(DMSetDimension(rdm, 2));
3649     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3650   } break;
3651   case DM_POLYTOPE_SEG_PRISM_TENSOR: {
3652     PetscInt    numPoints[2]        = {4, 1};
3653     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3654     PetscInt    cones[4]            = {1, 2, 3, 4};
3655     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3656     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0};
3657 
3658     PetscCall(DMSetDimension(rdm, 2));
3659     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3660   } break;
3661   case DM_POLYTOPE_TETRAHEDRON: {
3662     PetscInt    numPoints[2]        = {4, 1};
3663     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3664     PetscInt    cones[4]            = {1, 2, 3, 4};
3665     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3666     PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0};
3667 
3668     PetscCall(DMSetDimension(rdm, 3));
3669     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3670   } break;
3671   case DM_POLYTOPE_HEXAHEDRON: {
3672     PetscInt    numPoints[2]        = {8, 1};
3673     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3674     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3675     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3676     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3677 
3678     PetscCall(DMSetDimension(rdm, 3));
3679     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3680   } break;
3681   case DM_POLYTOPE_TRI_PRISM: {
3682     PetscInt    numPoints[2]        = {6, 1};
3683     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3684     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3685     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3686     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3687 
3688     PetscCall(DMSetDimension(rdm, 3));
3689     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3690   } break;
3691   case DM_POLYTOPE_TRI_PRISM_TENSOR: {
3692     PetscInt    numPoints[2]        = {6, 1};
3693     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3694     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3695     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3696     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3697 
3698     PetscCall(DMSetDimension(rdm, 3));
3699     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3700   } break;
3701   case DM_POLYTOPE_QUAD_PRISM_TENSOR: {
3702     PetscInt    numPoints[2]        = {8, 1};
3703     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3704     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3705     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3706     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3707 
3708     PetscCall(DMSetDimension(rdm, 3));
3709     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3710   } break;
3711   case DM_POLYTOPE_PYRAMID: {
3712     PetscInt    numPoints[2]        = {5, 1};
3713     PetscInt    coneSize[6]         = {5, 0, 0, 0, 0, 0};
3714     PetscInt    cones[5]            = {1, 2, 3, 4, 5};
3715     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3716     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, 0.0, 0.0, 1.0};
3717 
3718     PetscCall(DMSetDimension(rdm, 3));
3719     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3720   } break;
3721   default:
3722     SETERRQ(PetscObjectComm((PetscObject)rdm), PETSC_ERR_ARG_WRONG, "Cannot create reference cell for cell type %s", DMPolytopeTypes[ct]);
3723   }
3724   {
3725     PetscInt Nv, v;
3726 
3727     /* Must create the celltype label here so that we do not automatically try to compute the types */
3728     PetscCall(DMCreateLabel(rdm, "celltype"));
3729     PetscCall(DMPlexSetCellType(rdm, 0, ct));
3730     PetscCall(DMPlexGetChart(rdm, NULL, &Nv));
3731     for (v = 1; v < Nv; ++v) PetscCall(DMPlexSetCellType(rdm, v, DM_POLYTOPE_POINT));
3732   }
3733   PetscCall(DMPlexInterpolateInPlace_Internal(rdm));
3734   PetscCall(PetscObjectSetName((PetscObject)rdm, DMPolytopeTypes[ct]));
3735   PetscFunctionReturn(0);
3736 }
3737 
3738 /*@
3739   DMPlexCreateReferenceCell - Create a `DMPLEX` with the appropriate FEM reference cell
3740 
3741   Collective
3742 
3743   Input Parameters:
3744 + comm - The communicator
3745 - ct   - The cell type of the reference cell
3746 
3747   Output Parameter:
3748 . refdm - The reference cell
3749 
3750   Level: intermediate
3751 
3752 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateReferenceCell()`, `DMPlexCreateBoxMesh()`
3753 @*/
3754 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, DMPolytopeType ct, DM *refdm)
3755 {
3756   PetscFunctionBegin;
3757   PetscCall(DMCreate(comm, refdm));
3758   PetscCall(DMSetType(*refdm, DMPLEX));
3759   PetscCall(DMPlexCreateReferenceCell_Internal(*refdm, ct));
3760   PetscFunctionReturn(0);
3761 }
3762 
3763 static PetscErrorCode DMPlexCreateBoundaryLabel_Private(DM dm, const char name[])
3764 {
3765   DM        plex;
3766   DMLabel   label;
3767   PetscBool hasLabel;
3768 
3769   PetscFunctionBegin;
3770   PetscCall(DMHasLabel(dm, name, &hasLabel));
3771   if (hasLabel) PetscFunctionReturn(0);
3772   PetscCall(DMCreateLabel(dm, name));
3773   PetscCall(DMGetLabel(dm, name, &label));
3774   PetscCall(DMConvert(dm, DMPLEX, &plex));
3775   PetscCall(DMPlexMarkBoundaryFaces(plex, 1, label));
3776   PetscCall(DMPlexLabelComplete(plex, label));
3777   PetscCall(DMDestroy(&plex));
3778   PetscFunctionReturn(0);
3779 }
3780 
3781 /*
3782   We use the last coordinate as the radius, the inner radius is lower[dim-1] and the outer radius is upper[dim-1]. Then we map the first coordinate around the circle.
3783 
3784     (x, y) -> (r, theta) = (x[1], (x[0] - lower[0]) * 2\pi/(upper[0] - lower[0]))
3785 */
3786 static void boxToAnnulus(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
3787 {
3788   const PetscReal low = PetscRealPart(constants[0]);
3789   const PetscReal upp = PetscRealPart(constants[1]);
3790   const PetscReal r   = PetscRealPart(u[1]);
3791   const PetscReal th  = 2. * PETSC_PI * (PetscRealPart(u[0]) - low) / (upp - low);
3792 
3793   f0[0] = r * PetscCosReal(th);
3794   f0[1] = r * PetscSinReal(th);
3795 }
3796 
3797 const char *const DMPlexShapes[] = {"box", "box_surface", "ball", "sphere", "cylinder", "schwarz_p", "gyroid", "doublet", "annulus", "hypercubic", "unknown", "DMPlexShape", "DM_SHAPE_", NULL};
3798 
3799 static PetscErrorCode DMPlexCreateFromOptions_Internal(PetscOptionItems *PetscOptionsObject, PetscBool *useCoordSpace, DM dm)
3800 {
3801   DMPlexShape    shape   = DM_SHAPE_BOX;
3802   DMPolytopeType cell    = DM_POLYTOPE_TRIANGLE;
3803   PetscInt       dim     = 2;
3804   PetscBool      simplex = PETSC_TRUE, interpolate = PETSC_TRUE, adjCone = PETSC_FALSE, adjClosure = PETSC_TRUE, refDomain = PETSC_FALSE;
3805   PetscBool      flg, flg2, fflg, bdfflg, nameflg;
3806   MPI_Comm       comm;
3807   char           filename[PETSC_MAX_PATH_LEN]   = "<unspecified>";
3808   char           bdFilename[PETSC_MAX_PATH_LEN] = "<unspecified>";
3809   char           plexname[PETSC_MAX_PATH_LEN]   = "";
3810 
3811   PetscFunctionBegin;
3812   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3813   /* TODO Turn this into a registration interface */
3814   PetscCall(PetscOptionsString("-dm_plex_filename", "File containing a mesh", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &fflg));
3815   PetscCall(PetscOptionsString("-dm_plex_boundary_filename", "File containing a mesh boundary", "DMPlexCreateFromFile", bdFilename, bdFilename, sizeof(bdFilename), &bdfflg));
3816   PetscCall(PetscOptionsString("-dm_plex_name", "Name of the mesh in the file", "DMPlexCreateFromFile", plexname, plexname, sizeof(plexname), &nameflg));
3817   PetscCall(PetscOptionsEnum("-dm_plex_cell", "Cell shape", "", DMPolytopeTypes, (PetscEnum)cell, (PetscEnum *)&cell, NULL));
3818   PetscCall(PetscOptionsBool("-dm_plex_reference_cell_domain", "Use a reference cell domain", "", refDomain, &refDomain, NULL));
3819   PetscCall(PetscOptionsEnum("-dm_plex_shape", "Shape for built-in mesh", "", DMPlexShapes, (PetscEnum)shape, (PetscEnum *)&shape, &flg));
3820   PetscCall(PetscOptionsBoundedInt("-dm_plex_dim", "Topological dimension of the mesh", "DMGetDimension", dim, &dim, &flg, 0));
3821   PetscCheck(dim >= 0, comm, PETSC_ERR_ARG_OUTOFRANGE, "Dimension %" PetscInt_FMT " should be in [0, infinity)", dim);
3822   PetscCall(PetscOptionsBool("-dm_plex_simplex", "Mesh cell shape", "", simplex, &simplex, &flg));
3823   PetscCall(PetscOptionsBool("-dm_plex_interpolate", "Flag to create edges and faces automatically", "", interpolate, &interpolate, &flg));
3824   PetscCall(PetscOptionsBool("-dm_plex_adj_cone", "Set adjacency direction", "DMSetBasicAdjacency", adjCone, &adjCone, &flg));
3825   PetscCall(PetscOptionsBool("-dm_plex_adj_closure", "Set adjacency size", "DMSetBasicAdjacency", adjClosure, &adjClosure, &flg2));
3826   if (flg || flg2) PetscCall(DMSetBasicAdjacency(dm, adjCone, adjClosure));
3827 
3828   switch (cell) {
3829   case DM_POLYTOPE_POINT:
3830   case DM_POLYTOPE_SEGMENT:
3831   case DM_POLYTOPE_POINT_PRISM_TENSOR:
3832   case DM_POLYTOPE_TRIANGLE:
3833   case DM_POLYTOPE_QUADRILATERAL:
3834   case DM_POLYTOPE_TETRAHEDRON:
3835   case DM_POLYTOPE_HEXAHEDRON:
3836     *useCoordSpace = PETSC_TRUE;
3837     break;
3838   default:
3839     *useCoordSpace = PETSC_FALSE;
3840     break;
3841   }
3842 
3843   if (fflg) {
3844     DM dmnew;
3845 
3846     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), filename, plexname, interpolate, &dmnew));
3847     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3848     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3849   } else if (refDomain) {
3850     PetscCall(DMPlexCreateReferenceCell_Internal(dm, cell));
3851   } else if (bdfflg) {
3852     DM bdm, dmnew;
3853 
3854     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), bdFilename, plexname, interpolate, &bdm));
3855     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)bdm, "bd_"));
3856     PetscCall(DMSetFromOptions(bdm));
3857     PetscCall(DMPlexGenerate(bdm, NULL, interpolate, &dmnew));
3858     PetscCall(DMDestroy(&bdm));
3859     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3860     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3861   } else {
3862     PetscCall(PetscObjectSetName((PetscObject)dm, DMPlexShapes[shape]));
3863     switch (shape) {
3864     case DM_SHAPE_BOX:
3865     case DM_SHAPE_ANNULUS: {
3866       PetscInt       faces[3]  = {0, 0, 0};
3867       PetscReal      lower[3]  = {0, 0, 0};
3868       PetscReal      upper[3]  = {1, 1, 1};
3869       DMBoundaryType bdt[3]    = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3870       PetscBool      isAnnular = shape == DM_SHAPE_ANNULUS ? PETSC_TRUE : PETSC_FALSE;
3871       PetscInt       i, n;
3872 
3873       n = dim;
3874       for (i = 0; i < dim; ++i) faces[i] = (dim == 1 ? 1 : 4 - dim);
3875       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3876       n = 3;
3877       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3878       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3879       n = 3;
3880       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3881       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3882       n = 3;
3883       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
3884       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3885 
3886       PetscCheck(!isAnnular || dim == 2, comm, PETSC_ERR_ARG_OUTOFRANGE, "Only two dimensional annuli have been implemented");
3887       if (isAnnular)
3888         for (i = 0; i < dim - 1; ++i) bdt[i] = DM_BOUNDARY_PERIODIC;
3889 
3890       switch (cell) {
3891       case DM_POLYTOPE_TRI_PRISM_TENSOR:
3892         PetscCall(DMPlexCreateWedgeBoxMesh_Internal(dm, faces, lower, upper, bdt));
3893         if (!interpolate) {
3894           DM udm;
3895 
3896           PetscCall(DMPlexUninterpolate(dm, &udm));
3897           PetscCall(DMPlexReplace_Internal(dm, &udm));
3898         }
3899         break;
3900       default:
3901         PetscCall(DMPlexCreateBoxMesh_Internal(dm, dim, simplex, faces, lower, upper, bdt, interpolate));
3902         break;
3903       }
3904       if (isAnnular) {
3905         DM          cdm;
3906         PetscDS     cds;
3907         PetscScalar bounds[2] = {lower[0], upper[0]};
3908 
3909         // Fix coordinates for annular region
3910         PetscCall(DMSetPeriodicity(dm, NULL, NULL, NULL));
3911         PetscCall(DMSetCellCoordinatesLocal(dm, NULL));
3912         PetscCall(DMSetCellCoordinates(dm, NULL));
3913         PetscCall(DMPlexCreateCoordinateSpace(dm, 1, NULL));
3914         PetscCall(DMGetCoordinateDM(dm, &cdm));
3915         PetscCall(DMGetDS(cdm, &cds));
3916         PetscCall(PetscDSSetConstants(cds, 2, bounds));
3917         PetscCall(DMPlexRemapGeometry(dm, 0.0, boxToAnnulus));
3918       }
3919     } break;
3920     case DM_SHAPE_BOX_SURFACE: {
3921       PetscInt  faces[3] = {0, 0, 0};
3922       PetscReal lower[3] = {0, 0, 0};
3923       PetscReal upper[3] = {1, 1, 1};
3924       PetscInt  i, n;
3925 
3926       n = dim + 1;
3927       for (i = 0; i < dim + 1; ++i) faces[i] = (dim + 1 == 1 ? 1 : 4 - (dim + 1));
3928       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3929       n = 3;
3930       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3931       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3932       n = 3;
3933       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3934       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3935       PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(dm, dim + 1, faces, lower, upper, interpolate));
3936     } break;
3937     case DM_SHAPE_SPHERE: {
3938       PetscReal R = 1.0;
3939 
3940       PetscCall(PetscOptionsReal("-dm_plex_sphere_radius", "Radius of the sphere", "", R, &R, &flg));
3941       PetscCall(DMPlexCreateSphereMesh_Internal(dm, dim, simplex, R));
3942     } break;
3943     case DM_SHAPE_BALL: {
3944       PetscReal R = 1.0;
3945 
3946       PetscCall(PetscOptionsReal("-dm_plex_ball_radius", "Radius of the ball", "", R, &R, &flg));
3947       PetscCall(DMPlexCreateBallMesh_Internal(dm, dim, R));
3948     } break;
3949     case DM_SHAPE_CYLINDER: {
3950       DMBoundaryType bdt = DM_BOUNDARY_NONE;
3951       PetscInt       Nw  = 6;
3952 
3953       PetscCall(PetscOptionsEnum("-dm_plex_cylinder_bd", "Boundary type in the z direction", "", DMBoundaryTypes, (PetscEnum)bdt, (PetscEnum *)&bdt, NULL));
3954       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_wedges", "Number of wedges around the cylinder", "", Nw, &Nw, NULL));
3955       switch (cell) {
3956       case DM_POLYTOPE_TRI_PRISM_TENSOR:
3957         PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(dm, Nw, interpolate));
3958         break;
3959       default:
3960         PetscCall(DMPlexCreateHexCylinderMesh_Internal(dm, bdt));
3961         break;
3962       }
3963     } break;
3964     case DM_SHAPE_SCHWARZ_P: // fallthrough
3965     case DM_SHAPE_GYROID: {
3966       PetscInt       extent[3] = {1, 1, 1}, refine = 0, layers = 0, three;
3967       PetscReal      thickness   = 0.;
3968       DMBoundaryType periodic[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3969       DMPlexTPSType  tps_type    = shape == DM_SHAPE_SCHWARZ_P ? DMPLEX_TPS_SCHWARZ_P : DMPLEX_TPS_GYROID;
3970       PetscBool      tps_distribute;
3971       PetscCall(PetscOptionsIntArray("-dm_plex_tps_extent", "Number of replicas for each of three dimensions", NULL, extent, (three = 3, &three), NULL));
3972       PetscCall(PetscOptionsInt("-dm_plex_tps_refine", "Number of refinements", NULL, refine, &refine, NULL));
3973       PetscCall(PetscOptionsEnumArray("-dm_plex_tps_periodic", "Periodicity in each of three dimensions", NULL, DMBoundaryTypes, (PetscEnum *)periodic, (three = 3, &three), NULL));
3974       PetscCall(PetscOptionsInt("-dm_plex_tps_layers", "Number of layers in volumetric extrusion (or zero to not extrude)", NULL, layers, &layers, NULL));
3975       PetscCall(PetscOptionsReal("-dm_plex_tps_thickness", "Thickness of volumetric extrusion", NULL, thickness, &thickness, NULL));
3976       PetscCall(DMPlexDistributeGetDefault(dm, &tps_distribute));
3977       PetscCall(PetscOptionsBool("-dm_plex_tps_distribute", "Distribute the 2D mesh prior to refinement and extrusion", NULL, tps_distribute, &tps_distribute, NULL));
3978       PetscCall(DMPlexCreateTPSMesh_Internal(dm, tps_type, extent, periodic, tps_distribute, refine, layers, thickness));
3979     } break;
3980     case DM_SHAPE_DOUBLET: {
3981       DM        dmnew;
3982       PetscReal rl = 0.0;
3983 
3984       PetscCall(PetscOptionsReal("-dm_plex_doublet_refinementlimit", "Refinement limit", NULL, rl, &rl, NULL));
3985       PetscCall(DMPlexCreateDoublet(PetscObjectComm((PetscObject)dm), dim, simplex, interpolate, rl, &dmnew));
3986       PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3987       PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3988     } break;
3989     case DM_SHAPE_HYPERCUBIC: {
3990       PetscInt       *edges;
3991       PetscReal      *lower, *upper;
3992       DMBoundaryType *bdt;
3993       PetscInt        n, d;
3994 
3995       *useCoordSpace = PETSC_FALSE;
3996       PetscCall(PetscMalloc4(dim, &edges, dim, &lower, dim, &upper, dim, &bdt));
3997       for (d = 0; d < dim; ++d) {
3998         edges[d] = 1;
3999         lower[d] = 0.;
4000         upper[d] = 1.;
4001         bdt[d]   = DM_BOUNDARY_PERIODIC;
4002       }
4003       n = dim;
4004       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", edges, &n, &flg));
4005       n = dim;
4006       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
4007       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4008       n = dim;
4009       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
4010       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4011       n = dim;
4012       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
4013       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4014       PetscCall(DMPlexCreateHypercubicMesh_Internal(dm, dim, lower, upper, edges, bdt));
4015       PetscCall(PetscFree4(edges, lower, upper, bdt));
4016     } break;
4017     default:
4018       SETERRQ(comm, PETSC_ERR_SUP, "Domain shape %s is unsupported", DMPlexShapes[shape]);
4019     }
4020   }
4021   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
4022   if (!((PetscObject)dm)->name && nameflg) PetscCall(PetscObjectSetName((PetscObject)dm, plexname));
4023   PetscFunctionReturn(0);
4024 }
4025 
4026 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
4027 {
4028   DM_Plex  *mesh = (DM_Plex *)dm->data;
4029   PetscBool flg, flg2;
4030   char      bdLabel[PETSC_MAX_PATH_LEN];
4031 
4032   PetscFunctionBegin;
4033   /* Handle viewing */
4034   PetscCall(PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL));
4035   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fem", "Debug output level all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL, 0));
4036   PetscCall(PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL));
4037   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL, 0));
4038   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_locate", "Debug output level all point location computations", "DMLocatePoints", 0, &mesh->printLocate, NULL, 0));
4039   PetscCall(DMMonitorSetFromOptions(dm, "-dm_plex_monitor_throughput", "Monitor the simulation throughput", "DMPlexMonitorThroughput", DMPlexMonitorThroughput, NULL, &flg));
4040   if (flg) PetscCall(PetscLogDefaultBegin());
4041   /* Labeling */
4042   PetscCall(PetscOptionsString("-dm_plex_boundary_label", "Label to mark the mesh boundary", "", bdLabel, bdLabel, sizeof(bdLabel), &flg));
4043   if (flg) PetscCall(DMPlexCreateBoundaryLabel_Private(dm, bdLabel));
4044   /* Point Location */
4045   PetscCall(PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL));
4046   /* Partitioning and distribution */
4047   PetscCall(PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL));
4048   /* Generation and remeshing */
4049   PetscCall(PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL));
4050   /* Projection behavior */
4051   PetscCall(PetscOptionsBoundedInt("-dm_plex_max_projection_height", "Maximum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL, 0));
4052   PetscCall(PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL));
4053   /* Checking structure */
4054   {
4055     PetscBool all = PETSC_FALSE;
4056 
4057     PetscCall(PetscOptionsBool("-dm_plex_check_all", "Perform all basic checks", "DMPlexCheck", PETSC_FALSE, &all, NULL));
4058     if (all) {
4059       PetscCall(DMPlexCheck(dm));
4060     } else {
4061       PetscCall(PetscOptionsBool("-dm_plex_check_symmetry", "Check that the adjacency information in the mesh is symmetric", "DMPlexCheckSymmetry", PETSC_FALSE, &flg, &flg2));
4062       if (flg && flg2) PetscCall(DMPlexCheckSymmetry(dm));
4063       PetscCall(PetscOptionsBool("-dm_plex_check_skeleton", "Check that each cell has the correct number of vertices (only for homogeneous simplex or tensor meshes)", "DMPlexCheckSkeleton", PETSC_FALSE, &flg, &flg2));
4064       if (flg && flg2) PetscCall(DMPlexCheckSkeleton(dm, 0));
4065       PetscCall(PetscOptionsBool("-dm_plex_check_faces", "Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type", "DMPlexCheckFaces", PETSC_FALSE, &flg, &flg2));
4066       if (flg && flg2) PetscCall(DMPlexCheckFaces(dm, 0));
4067       PetscCall(PetscOptionsBool("-dm_plex_check_geometry", "Check that cells have positive volume", "DMPlexCheckGeometry", PETSC_FALSE, &flg, &flg2));
4068       if (flg && flg2) PetscCall(DMPlexCheckGeometry(dm));
4069       PetscCall(PetscOptionsBool("-dm_plex_check_pointsf", "Check some necessary conditions for PointSF", "DMPlexCheckPointSF", PETSC_FALSE, &flg, &flg2));
4070       if (flg && flg2) PetscCall(DMPlexCheckPointSF(dm, NULL, PETSC_FALSE));
4071       PetscCall(PetscOptionsBool("-dm_plex_check_interface_cones", "Check points on inter-partition interfaces have conforming order of cone points", "DMPlexCheckInterfaceCones", PETSC_FALSE, &flg, &flg2));
4072       if (flg && flg2) PetscCall(DMPlexCheckInterfaceCones(dm));
4073     }
4074     PetscCall(PetscOptionsBool("-dm_plex_check_cell_shape", "Check cell shape", "DMPlexCheckCellShape", PETSC_FALSE, &flg, &flg2));
4075     if (flg && flg2) PetscCall(DMPlexCheckCellShape(dm, PETSC_TRUE, PETSC_DETERMINE));
4076   }
4077   {
4078     PetscReal scale = 1.0;
4079 
4080     PetscCall(PetscOptionsReal("-dm_plex_scale", "Scale factor for mesh coordinates", "DMPlexScale", scale, &scale, &flg));
4081     if (flg) {
4082       Vec coordinates, coordinatesLocal;
4083 
4084       PetscCall(DMGetCoordinates(dm, &coordinates));
4085       PetscCall(DMGetCoordinatesLocal(dm, &coordinatesLocal));
4086       PetscCall(VecScale(coordinates, scale));
4087       PetscCall(VecScale(coordinatesLocal, scale));
4088     }
4089   }
4090   PetscCall(PetscPartitionerSetFromOptions(mesh->partitioner));
4091   PetscFunctionReturn(0);
4092 }
4093 
4094 PetscErrorCode DMSetFromOptions_Overlap_Plex(DM dm, PetscOptionItems *PetscOptionsObject, PetscInt *overlap)
4095 {
4096   PetscInt  numOvLabels = 16, numOvExLabels = 16;
4097   char     *ovLabelNames[16], *ovExLabelNames[16];
4098   PetscInt  numOvValues = 16, numOvExValues = 16, l;
4099   PetscBool flg;
4100 
4101   PetscFunctionBegin;
4102   PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", *overlap, overlap, NULL, 0));
4103   PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_labels", "List of overlap label names", "DMPlexDistribute", ovLabelNames, &numOvLabels, &flg));
4104   if (!flg) numOvLabels = 0;
4105   if (numOvLabels) {
4106     ((DM_Plex *)dm->data)->numOvLabels = numOvLabels;
4107     for (l = 0; l < numOvLabels; ++l) {
4108       PetscCall(DMGetLabel(dm, ovLabelNames[l], &((DM_Plex *)dm->data)->ovLabels[l]));
4109       PetscCheck(((DM_Plex *)dm->data)->ovLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovLabelNames[l]);
4110       PetscCall(PetscFree(ovLabelNames[l]));
4111     }
4112     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_values", "List of overlap label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovValues, &numOvValues, &flg));
4113     if (!flg) numOvValues = 0;
4114     PetscCheck(numOvLabels == numOvValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvLabels, numOvValues);
4115 
4116     PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_exclude_labels", "List of overlap exclude label names", "DMPlexDistribute", ovExLabelNames, &numOvExLabels, &flg));
4117     if (!flg) numOvExLabels = 0;
4118     ((DM_Plex *)dm->data)->numOvExLabels = numOvExLabels;
4119     for (l = 0; l < numOvExLabels; ++l) {
4120       PetscCall(DMGetLabel(dm, ovExLabelNames[l], &((DM_Plex *)dm->data)->ovExLabels[l]));
4121       PetscCheck(((DM_Plex *)dm->data)->ovExLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovExLabelNames[l]);
4122       PetscCall(PetscFree(ovExLabelNames[l]));
4123     }
4124     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_exclude_values", "List of overlap exclude label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovExValues, &numOvExValues, &flg));
4125     if (!flg) numOvExValues = 0;
4126     PetscCheck(numOvExLabels == numOvExValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of exclude labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvExLabels, numOvExValues);
4127   }
4128   PetscFunctionReturn(0);
4129 }
4130 
4131 static PetscErrorCode DMSetFromOptions_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
4132 {
4133   PetscFunctionList        ordlist;
4134   char                     oname[256];
4135   PetscReal                volume    = -1.0;
4136   PetscInt                 prerefine = 0, refine = 0, r, coarsen = 0, overlap = 0, extLayers = 0, dim;
4137   PetscBool                uniformOrig, created = PETSC_FALSE, uniform = PETSC_TRUE, distribute, interpolate = PETSC_TRUE, coordSpace = PETSC_TRUE, remap = PETSC_TRUE, ghostCells = PETSC_FALSE, isHierarchy, ignoreModel = PETSC_FALSE, flg;
4138   DMPlexReorderDefaultFlag reorder;
4139 
4140   PetscFunctionBegin;
4141   PetscOptionsHeadBegin(PetscOptionsObject, "DMPlex Options");
4142   if (dm->cloneOpts) goto non_refine;
4143   /* Handle automatic creation */
4144   PetscCall(DMGetDimension(dm, &dim));
4145   if (dim < 0) {
4146     PetscCall(DMPlexCreateFromOptions_Internal(PetscOptionsObject, &coordSpace, dm));
4147     created = PETSC_TRUE;
4148   }
4149   PetscCall(DMGetDimension(dm, &dim));
4150   /* Handle interpolation before distribution */
4151   PetscCall(PetscOptionsBool("-dm_plex_interpolate_pre", "Flag to interpolate mesh before distribution", "", interpolate, &interpolate, &flg));
4152   if (flg) {
4153     DMPlexInterpolatedFlag interpolated;
4154 
4155     PetscCall(DMPlexIsInterpolated(dm, &interpolated));
4156     if (interpolated == DMPLEX_INTERPOLATED_FULL && !interpolate) {
4157       DM udm;
4158 
4159       PetscCall(DMPlexUninterpolate(dm, &udm));
4160       PetscCall(DMPlexReplace_Internal(dm, &udm));
4161     } else if (interpolated != DMPLEX_INTERPOLATED_FULL && interpolate) {
4162       DM idm;
4163 
4164       PetscCall(DMPlexInterpolate(dm, &idm));
4165       PetscCall(DMPlexReplace_Internal(dm, &idm));
4166     }
4167   }
4168   /* Handle DMPlex refinement before distribution */
4169   PetscCall(PetscOptionsBool("-dm_refine_ignore_model", "Flag to ignore the geometry model when refining", "DMCreate", ignoreModel, &ignoreModel, &flg));
4170   if (flg) ((DM_Plex *)dm->data)->ignoreModel = ignoreModel;
4171   PetscCall(DMPlexGetRefinementUniform(dm, &uniformOrig));
4172   PetscCall(PetscOptionsBoundedInt("-dm_refine_pre", "The number of refinements before distribution", "DMCreate", prerefine, &prerefine, NULL, 0));
4173   PetscCall(PetscOptionsBool("-dm_refine_remap_pre", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
4174   PetscCall(PetscOptionsBool("-dm_refine_uniform_pre", "Flag for uniform refinement before distribution", "DMCreate", uniform, &uniform, &flg));
4175   if (flg) PetscCall(DMPlexSetRefinementUniform(dm, uniform));
4176   PetscCall(PetscOptionsReal("-dm_refine_volume_limit_pre", "The maximum cell volume after refinement before distribution", "DMCreate", volume, &volume, &flg));
4177   if (flg) {
4178     PetscCall(DMPlexSetRefinementUniform(dm, PETSC_FALSE));
4179     PetscCall(DMPlexSetRefinementLimit(dm, volume));
4180     prerefine = PetscMax(prerefine, 1);
4181   }
4182   for (r = 0; r < prerefine; ++r) {
4183     DM             rdm;
4184     PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4185 
4186     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4187     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
4188     PetscCall(DMPlexReplace_Internal(dm, &rdm));
4189     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4190     if (coordFunc && remap) {
4191       PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4192       ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4193     }
4194   }
4195   PetscCall(DMPlexSetRefinementUniform(dm, uniformOrig));
4196   /* Handle DMPlex extrusion before distribution */
4197   PetscCall(PetscOptionsBoundedInt("-dm_extrude", "The number of layers to extrude", "", extLayers, &extLayers, NULL, 0));
4198   if (extLayers) {
4199     DM edm;
4200 
4201     PetscCall(DMExtrude(dm, extLayers, &edm));
4202     PetscCall(DMPlexReplace_Internal(dm, &edm));
4203     ((DM_Plex *)dm->data)->coordFunc = NULL;
4204     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4205     extLayers = 0;
4206     PetscCall(DMGetDimension(dm, &dim));
4207   }
4208   /* Handle DMPlex reordering before distribution */
4209   PetscCall(DMPlexReorderGetDefault(dm, &reorder));
4210   PetscCall(MatGetOrderingList(&ordlist));
4211   PetscCall(PetscStrncpy(oname, MATORDERINGNATURAL, sizeof(oname)));
4212   PetscCall(PetscOptionsFList("-dm_plex_reorder", "Set mesh reordering type", "DMPlexGetOrdering", ordlist, MATORDERINGNATURAL, oname, sizeof(oname), &flg));
4213   if (reorder == DMPLEX_REORDER_DEFAULT_TRUE || flg) {
4214     DM pdm;
4215     IS perm;
4216 
4217     PetscCall(DMPlexGetOrdering(dm, oname, NULL, &perm));
4218     PetscCall(DMPlexPermute(dm, perm, &pdm));
4219     PetscCall(ISDestroy(&perm));
4220     PetscCall(DMPlexReplace_Internal(dm, &pdm));
4221     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4222   }
4223   /* Handle DMPlex distribution */
4224   PetscCall(DMPlexDistributeGetDefault(dm, &distribute));
4225   PetscCall(PetscOptionsBool("-dm_distribute", "Flag to redistribute a mesh among processes", "DMPlexDistribute", distribute, &distribute, NULL));
4226   PetscCall(DMSetFromOptions_Overlap_Plex(dm, PetscOptionsObject, &overlap));
4227   if (distribute) {
4228     DM               pdm = NULL;
4229     PetscPartitioner part;
4230 
4231     PetscCall(DMPlexGetPartitioner(dm, &part));
4232     PetscCall(PetscPartitionerSetFromOptions(part));
4233     PetscCall(DMPlexDistribute(dm, overlap, NULL, &pdm));
4234     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
4235   }
4236   /* Create coordinate space */
4237   if (created) {
4238     DM_Plex  *mesh   = (DM_Plex *)dm->data;
4239     PetscInt  degree = 1;
4240     PetscInt  height = 0;
4241     DM        cdm;
4242     PetscBool flg;
4243 
4244     PetscCall(PetscOptionsBool("-dm_coord_space", "Use an FEM space for coordinates", "", coordSpace, &coordSpace, &flg));
4245     PetscCall(PetscOptionsInt("-dm_coord_petscspace_degree", "FEM degree for coordinate space", "", degree, &degree, NULL));
4246     if (coordSpace) PetscCall(DMPlexCreateCoordinateSpace(dm, degree, mesh->coordFunc));
4247     PetscCall(DMGetCoordinateDM(dm, &cdm));
4248     if (flg && !coordSpace) {
4249       PetscDS      cds;
4250       PetscObject  obj;
4251       PetscClassId id;
4252 
4253       PetscCall(DMGetDS(cdm, &cds));
4254       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
4255       PetscCall(PetscObjectGetClassId(obj, &id));
4256       if (id == PETSCFE_CLASSID) {
4257         PetscContainer dummy;
4258 
4259         PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &dummy));
4260         PetscCall(PetscObjectSetName((PetscObject)dummy, "coordinates"));
4261         PetscCall(DMSetField(cdm, 0, NULL, (PetscObject)dummy));
4262         PetscCall(PetscContainerDestroy(&dummy));
4263         PetscCall(DMClearDS(cdm));
4264       }
4265       mesh->coordFunc = NULL;
4266     }
4267     PetscCall(PetscOptionsBool("-dm_sparse_localize", "Localize only necessary cells", "", dm->sparseLocalize, &dm->sparseLocalize, &flg));
4268     PetscCall(PetscOptionsInt("-dm_localize_height", "Localize edges and faces in addition to cells", "", height, &height, &flg));
4269     if (flg) PetscCall(DMPlexSetMaxProjectionHeight(cdm, height));
4270     PetscCall(DMLocalizeCoordinates(dm));
4271   }
4272   /* Handle DMPlex refinement */
4273   remap = PETSC_TRUE;
4274   PetscCall(PetscOptionsBoundedInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL, 0));
4275   PetscCall(PetscOptionsBool("-dm_refine_remap", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
4276   PetscCall(PetscOptionsBoundedInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy, 0));
4277   if (refine) PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
4278   if (refine && isHierarchy) {
4279     DM *dms, coarseDM;
4280 
4281     PetscCall(DMGetCoarseDM(dm, &coarseDM));
4282     PetscCall(PetscObjectReference((PetscObject)coarseDM));
4283     PetscCall(PetscMalloc1(refine, &dms));
4284     PetscCall(DMRefineHierarchy(dm, refine, dms));
4285     /* Total hack since we do not pass in a pointer */
4286     PetscCall(DMPlexSwap_Static(dm, dms[refine - 1]));
4287     if (refine == 1) {
4288       PetscCall(DMSetCoarseDM(dm, dms[0]));
4289       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4290     } else {
4291       PetscCall(DMSetCoarseDM(dm, dms[refine - 2]));
4292       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4293       PetscCall(DMSetCoarseDM(dms[0], dms[refine - 1]));
4294       PetscCall(DMPlexSetRegularRefinement(dms[0], PETSC_TRUE));
4295     }
4296     PetscCall(DMSetCoarseDM(dms[refine - 1], coarseDM));
4297     PetscCall(PetscObjectDereference((PetscObject)coarseDM));
4298     /* Free DMs */
4299     for (r = 0; r < refine; ++r) {
4300       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4301       PetscCall(DMDestroy(&dms[r]));
4302     }
4303     PetscCall(PetscFree(dms));
4304   } else {
4305     for (r = 0; r < refine; ++r) {
4306       DM             rdm;
4307       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4308 
4309       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4310       PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
4311       /* Total hack since we do not pass in a pointer */
4312       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4313       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4314       if (coordFunc && remap) {
4315         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4316         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4317       }
4318     }
4319   }
4320   /* Handle DMPlex coarsening */
4321   PetscCall(PetscOptionsBoundedInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL, 0));
4322   PetscCall(PetscOptionsBoundedInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy, 0));
4323   if (coarsen && isHierarchy) {
4324     DM *dms;
4325 
4326     PetscCall(PetscMalloc1(coarsen, &dms));
4327     PetscCall(DMCoarsenHierarchy(dm, coarsen, dms));
4328     /* Free DMs */
4329     for (r = 0; r < coarsen; ++r) {
4330       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4331       PetscCall(DMDestroy(&dms[r]));
4332     }
4333     PetscCall(PetscFree(dms));
4334   } else {
4335     for (r = 0; r < coarsen; ++r) {
4336       DM             cdm;
4337       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4338 
4339       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4340       PetscCall(DMCoarsen(dm, PetscObjectComm((PetscObject)dm), &cdm));
4341       /* Total hack since we do not pass in a pointer */
4342       PetscCall(DMPlexReplace_Internal(dm, &cdm));
4343       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4344       if (coordFunc) {
4345         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4346         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4347       }
4348     }
4349   }
4350   /* Handle ghost cells */
4351   PetscCall(PetscOptionsBool("-dm_plex_create_fv_ghost_cells", "Flag to create finite volume ghost cells on the boundary", "DMCreate", ghostCells, &ghostCells, NULL));
4352   if (ghostCells) {
4353     DM   gdm;
4354     char lname[PETSC_MAX_PATH_LEN];
4355 
4356     lname[0] = '\0';
4357     PetscCall(PetscOptionsString("-dm_plex_fv_ghost_cells_label", "Label name for ghost cells boundary", "DMCreate", lname, lname, sizeof(lname), &flg));
4358     PetscCall(DMPlexConstructGhostCells(dm, flg ? lname : NULL, NULL, &gdm));
4359     PetscCall(DMPlexReplace_Internal(dm, &gdm));
4360   }
4361   /* Handle 1D order */
4362   if (reorder != DMPLEX_REORDER_DEFAULT_FALSE && dim == 1) {
4363     DM           cdm, rdm;
4364     PetscDS      cds;
4365     PetscObject  obj;
4366     PetscClassId id = PETSC_OBJECT_CLASSID;
4367     IS           perm;
4368     PetscInt     Nf;
4369     PetscBool    distributed;
4370 
4371     PetscCall(DMPlexIsDistributed(dm, &distributed));
4372     PetscCall(DMGetCoordinateDM(dm, &cdm));
4373     PetscCall(DMGetDS(cdm, &cds));
4374     PetscCall(PetscDSGetNumFields(cds, &Nf));
4375     if (Nf) {
4376       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
4377       PetscCall(PetscObjectGetClassId(obj, &id));
4378     }
4379     if (!distributed && id != PETSCFE_CLASSID) {
4380       PetscCall(DMPlexGetOrdering1D(dm, &perm));
4381       PetscCall(DMPlexPermute(dm, perm, &rdm));
4382       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4383       PetscCall(ISDestroy(&perm));
4384     }
4385   }
4386 /* Handle */
4387 non_refine:
4388   PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4389   PetscOptionsHeadEnd();
4390   PetscFunctionReturn(0);
4391 }
4392 
4393 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm, Vec *vec)
4394 {
4395   PetscFunctionBegin;
4396   PetscCall(DMCreateGlobalVector_Section_Private(dm, vec));
4397   /* PetscCall(VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM)); */
4398   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex));
4399   PetscCall(VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void))VecView_Plex_Native));
4400   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex));
4401   PetscCall(VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void))VecLoad_Plex_Native));
4402   PetscFunctionReturn(0);
4403 }
4404 
4405 static PetscErrorCode DMCreateLocalVector_Plex(DM dm, Vec *vec)
4406 {
4407   PetscFunctionBegin;
4408   PetscCall(DMCreateLocalVector_Section_Private(dm, vec));
4409   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex_Local));
4410   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex_Local));
4411   PetscFunctionReturn(0);
4412 }
4413 
4414 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
4415 {
4416   PetscInt depth, d;
4417 
4418   PetscFunctionBegin;
4419   PetscCall(DMPlexGetDepth(dm, &depth));
4420   if (depth == 1) {
4421     PetscCall(DMGetDimension(dm, &d));
4422     if (dim == 0) PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4423     else if (dim == d) PetscCall(DMPlexGetDepthStratum(dm, 1, pStart, pEnd));
4424     else {
4425       *pStart = 0;
4426       *pEnd   = 0;
4427     }
4428   } else {
4429     PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4430   }
4431   PetscFunctionReturn(0);
4432 }
4433 
4434 static PetscErrorCode DMGetNeighbors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
4435 {
4436   PetscSF            sf;
4437   PetscInt           niranks, njranks, n;
4438   const PetscMPIInt *iranks, *jranks;
4439   DM_Plex           *data = (DM_Plex *)dm->data;
4440 
4441   PetscFunctionBegin;
4442   PetscCall(DMGetPointSF(dm, &sf));
4443   if (!data->neighbors) {
4444     PetscCall(PetscSFSetUp(sf));
4445     PetscCall(PetscSFGetRootRanks(sf, &njranks, &jranks, NULL, NULL, NULL));
4446     PetscCall(PetscSFGetLeafRanks(sf, &niranks, &iranks, NULL, NULL));
4447     PetscCall(PetscMalloc1(njranks + niranks + 1, &data->neighbors));
4448     PetscCall(PetscArraycpy(data->neighbors + 1, jranks, njranks));
4449     PetscCall(PetscArraycpy(data->neighbors + njranks + 1, iranks, niranks));
4450     n = njranks + niranks;
4451     PetscCall(PetscSortRemoveDupsMPIInt(&n, data->neighbors + 1));
4452     /* The following cast should never fail: can't have more neighbors than PETSC_MPI_INT_MAX */
4453     PetscCall(PetscMPIIntCast(n, data->neighbors));
4454   }
4455   if (nranks) *nranks = data->neighbors[0];
4456   if (ranks) {
4457     if (data->neighbors[0]) *ranks = data->neighbors + 1;
4458     else *ranks = NULL;
4459   }
4460   PetscFunctionReturn(0);
4461 }
4462 
4463 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM, DM, Mat, Vec, Vec);
4464 
4465 static PetscErrorCode DMInitialize_Plex(DM dm)
4466 {
4467   PetscFunctionBegin;
4468   dm->ops->view                      = DMView_Plex;
4469   dm->ops->load                      = DMLoad_Plex;
4470   dm->ops->setfromoptions            = DMSetFromOptions_Plex;
4471   dm->ops->clone                     = DMClone_Plex;
4472   dm->ops->setup                     = DMSetUp_Plex;
4473   dm->ops->createlocalsection        = DMCreateLocalSection_Plex;
4474   dm->ops->createdefaultconstraints  = DMCreateDefaultConstraints_Plex;
4475   dm->ops->createglobalvector        = DMCreateGlobalVector_Plex;
4476   dm->ops->createlocalvector         = DMCreateLocalVector_Plex;
4477   dm->ops->getlocaltoglobalmapping   = NULL;
4478   dm->ops->createfieldis             = NULL;
4479   dm->ops->createcoordinatedm        = DMCreateCoordinateDM_Plex;
4480   dm->ops->createcoordinatefield     = DMCreateCoordinateField_Plex;
4481   dm->ops->getcoloring               = NULL;
4482   dm->ops->creatematrix              = DMCreateMatrix_Plex;
4483   dm->ops->createinterpolation       = DMCreateInterpolation_Plex;
4484   dm->ops->createmassmatrix          = DMCreateMassMatrix_Plex;
4485   dm->ops->createmassmatrixlumped    = DMCreateMassMatrixLumped_Plex;
4486   dm->ops->createinjection           = DMCreateInjection_Plex;
4487   dm->ops->refine                    = DMRefine_Plex;
4488   dm->ops->coarsen                   = DMCoarsen_Plex;
4489   dm->ops->refinehierarchy           = DMRefineHierarchy_Plex;
4490   dm->ops->coarsenhierarchy          = DMCoarsenHierarchy_Plex;
4491   dm->ops->extrude                   = DMExtrude_Plex;
4492   dm->ops->globaltolocalbegin        = NULL;
4493   dm->ops->globaltolocalend          = NULL;
4494   dm->ops->localtoglobalbegin        = NULL;
4495   dm->ops->localtoglobalend          = NULL;
4496   dm->ops->destroy                   = DMDestroy_Plex;
4497   dm->ops->createsubdm               = DMCreateSubDM_Plex;
4498   dm->ops->createsuperdm             = DMCreateSuperDM_Plex;
4499   dm->ops->getdimpoints              = DMGetDimPoints_Plex;
4500   dm->ops->locatepoints              = DMLocatePoints_Plex;
4501   dm->ops->projectfunctionlocal      = DMProjectFunctionLocal_Plex;
4502   dm->ops->projectfunctionlabellocal = DMProjectFunctionLabelLocal_Plex;
4503   dm->ops->projectfieldlocal         = DMProjectFieldLocal_Plex;
4504   dm->ops->projectfieldlabellocal    = DMProjectFieldLabelLocal_Plex;
4505   dm->ops->projectbdfieldlabellocal  = DMProjectBdFieldLabelLocal_Plex;
4506   dm->ops->computel2diff             = DMComputeL2Diff_Plex;
4507   dm->ops->computel2gradientdiff     = DMComputeL2GradientDiff_Plex;
4508   dm->ops->computel2fielddiff        = DMComputeL2FieldDiff_Plex;
4509   dm->ops->getneighbors              = DMGetNeighbors_Plex;
4510   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertBoundaryValues_C", DMPlexInsertBoundaryValues_Plex));
4511   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertTimeDerviativeBoundaryValues_C", DMPlexInsertTimeDerivativeBoundaryValues_Plex));
4512   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMSetUpGLVisViewer_C", DMSetUpGLVisViewer_Plex));
4513   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMCreateNeumannOverlap_C", DMCreateNeumannOverlap_Plex));
4514   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4515   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeGetDefault_C", DMPlexDistributeGetDefault_Plex));
4516   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeSetDefault_C", DMPlexDistributeSetDefault_Plex));
4517   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderGetDefault_C", DMPlexReorderGetDefault_Plex));
4518   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderSetDefault_C", DMPlexReorderSetDefault_Plex));
4519   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMInterpolateSolution_C", DMInterpolateSolution_Plex));
4520   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4521   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetOverlap_C", DMPlexSetOverlap_Plex));
4522   PetscFunctionReturn(0);
4523 }
4524 
4525 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
4526 {
4527   DM_Plex *mesh = (DM_Plex *)dm->data;
4528   PetscSF  face_sf;
4529 
4530   PetscFunctionBegin;
4531   mesh->refct++;
4532   (*newdm)->data = mesh;
4533   PetscCall(DMPlexGetIsoperiodicFaceSF(dm, &face_sf));
4534   PetscCall(DMPlexSetIsoperiodicFaceSF(*newdm, face_sf));
4535   PetscCall(PetscObjectChangeTypeName((PetscObject)*newdm, DMPLEX));
4536   PetscCall(DMInitialize_Plex(*newdm));
4537   PetscFunctionReturn(0);
4538 }
4539 
4540 /*MC
4541   DMPLEX = "plex" - A `DM` object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
4542                     In the local representation, Vecs contain all unknowns in the interior and shared boundary. This is
4543                     specified by a PetscSection object. Ownership in the global representation is determined by
4544                     ownership of the underlying `DMPLEX` points. This is specified by another `PetscSection` object.
4545 
4546   Options Database Keys:
4547 + -dm_refine_pre                     - Refine mesh before distribution
4548 + -dm_refine_uniform_pre             - Choose uniform or generator-based refinement
4549 + -dm_refine_volume_limit_pre        - Cell volume limit after pre-refinement using generator
4550 . -dm_distribute                     - Distribute mesh across processes
4551 . -dm_distribute_overlap             - Number of cells to overlap for distribution
4552 . -dm_refine                         - Refine mesh after distribution
4553 . -dm_plex_hash_location             - Use grid hashing for point location
4554 . -dm_plex_hash_box_faces <n,m,p>    - The number of divisions in each direction of the grid hash
4555 . -dm_plex_partition_balance         - Attempt to evenly divide points on partition boundary between processes
4556 . -dm_plex_remesh_bd                 - Allow changes to the boundary on remeshing
4557 . -dm_plex_max_projection_height     - Maximum mesh point height used to project locally
4558 . -dm_plex_regular_refinement        - Use special nested projection algorithm for regular refinement
4559 . -dm_plex_check_all                 - Perform all shecks below
4560 . -dm_plex_check_symmetry            - Check that the adjacency information in the mesh is symmetric
4561 . -dm_plex_check_skeleton <celltype> - Check that each cell has the correct number of vertices
4562 . -dm_plex_check_faces <celltype>    - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
4563 . -dm_plex_check_geometry            - Check that cells have positive volume
4564 . -dm_view :mesh.tex:ascii_latex     - View the mesh in LaTeX/TikZ
4565 . -dm_plex_view_scale <num>          - Scale the TikZ
4566 - -dm_plex_print_fem <num>           - View FEM assembly information, such as element vectors and matrices
4567 
4568   Level: intermediate
4569 
4570 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`, `PetscSection`
4571 M*/
4572 
4573 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
4574 {
4575   DM_Plex *mesh;
4576   PetscInt unit;
4577 
4578   PetscFunctionBegin;
4579   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4580   PetscCall(PetscNew(&mesh));
4581   dm->data = mesh;
4582 
4583   mesh->refct = 1;
4584   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection));
4585   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection));
4586   mesh->refinementUniform      = PETSC_TRUE;
4587   mesh->refinementLimit        = -1.0;
4588   mesh->distDefault            = PETSC_TRUE;
4589   mesh->reorderDefault         = DMPLEX_REORDER_DEFAULT_NOTSET;
4590   mesh->distributionName       = NULL;
4591   mesh->interpolated           = DMPLEX_INTERPOLATED_INVALID;
4592   mesh->interpolatedCollective = DMPLEX_INTERPOLATED_INVALID;
4593 
4594   PetscCall(PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner));
4595   mesh->remeshBd = PETSC_FALSE;
4596 
4597   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
4598 
4599   mesh->depthState    = -1;
4600   mesh->celltypeState = -1;
4601   mesh->printTol      = 1.0e-10;
4602 
4603   PetscCall(DMInitialize_Plex(dm));
4604   PetscFunctionReturn(0);
4605 }
4606 
4607 /*@
4608   DMPlexCreate - Creates a `DMPLEX` object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
4609 
4610   Collective
4611 
4612   Input Parameter:
4613 . comm - The communicator for the `DMPLEX` object
4614 
4615   Output Parameter:
4616 . mesh  - The `DMPLEX` object
4617 
4618   Level: beginner
4619 
4620 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`
4621 @*/
4622 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
4623 {
4624   PetscFunctionBegin;
4625   PetscValidPointer(mesh, 2);
4626   PetscCall(DMCreate(comm, mesh));
4627   PetscCall(DMSetType(*mesh, DMPLEX));
4628   PetscFunctionReturn(0);
4629 }
4630 
4631 /*@C
4632   DMPlexBuildFromCellListParallel - Build distributed `DMPLEX` topology from a list of vertices for each cell (common mesh generator output)
4633 
4634   Collective on dm
4635 
4636   Input Parameters:
4637 + dm - The `DM`
4638 . numCells - The number of cells owned by this process
4639 . numVertices - The number of vertices to be owned by this process, or `PETSC_DECIDE`
4640 . NVertices - The global number of vertices, or `PETSC_DETERMINE`
4641 . numCorners - The number of vertices for each cell
4642 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4643 
4644   Output Parameters:
4645 + vertexSF - (Optional) `PetscSF` describing complete vertex ownership
4646 - verticesAdjSaved - (Optional) vertex adjacency array
4647 
4648   Level: advanced
4649 
4650   Notes:
4651   Two triangles sharing a face
4652 .vb
4653 
4654         2
4655       / | \
4656      /  |  \
4657     /   |   \
4658    0  0 | 1  3
4659     \   |   /
4660      \  |  /
4661       \ | /
4662         1
4663 .ve
4664 would have input
4665 .vb
4666   numCells = 2, numVertices = 4
4667   cells = [0 1 2  1 3 2]
4668 .ve
4669 which would result in the `DMPLEX`
4670 .vb
4671 
4672         4
4673       / | \
4674      /  |  \
4675     /   |   \
4676    2  0 | 1  5
4677     \   |   /
4678      \  |  /
4679       \ | /
4680         3
4681 .ve
4682 
4683   Vertices are implicitly numbered consecutively 0,...,NVertices.
4684   Each rank owns a chunk of numVertices consecutive vertices.
4685   If numVertices is `PETSC_DECIDE`, PETSc will distribute them as evenly as possible using PetscLayout.
4686   If NVertices is `PETSC_DETERMINE` and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
4687   If only NVertices is `PETSC_DETERMINE`, it is computed as the sum of numVertices over all ranks.
4688 
4689   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
4690 
4691   Fortran Note:
4692   Not currently supported in Fortran.
4693 
4694 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildCoordinatesFromCellListParallel()`,
4695           `PetscSF`
4696 @*/
4697 PetscErrorCode DMPlexBuildFromCellListParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, const PetscInt cells[], PetscSF *vertexSF, PetscInt **verticesAdjSaved)
4698 {
4699   PetscSF     sfPoint;
4700   PetscLayout layout;
4701   PetscInt    numVerticesAdj, *verticesAdj, *cones, c, p;
4702 
4703   PetscFunctionBegin;
4704   PetscValidLogicalCollectiveInt(dm, NVertices, 4);
4705   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4706   /* Get/check global number of vertices */
4707   {
4708     PetscInt       NVerticesInCells, i;
4709     const PetscInt len = numCells * numCorners;
4710 
4711     /* NVerticesInCells = max(cells) + 1 */
4712     NVerticesInCells = PETSC_MIN_INT;
4713     for (i = 0; i < len; i++)
4714       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4715     ++NVerticesInCells;
4716     PetscCallMPI(MPI_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
4717 
4718     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
4719     else
4720       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
4721   }
4722   /* Count locally unique vertices */
4723   {
4724     PetscHSetI vhash;
4725     PetscInt   off = 0;
4726 
4727     PetscCall(PetscHSetICreate(&vhash));
4728     for (c = 0; c < numCells; ++c) {
4729       for (p = 0; p < numCorners; ++p) PetscCall(PetscHSetIAdd(vhash, cells[c * numCorners + p]));
4730     }
4731     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
4732     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
4733     else verticesAdj = *verticesAdjSaved;
4734     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
4735     PetscCall(PetscHSetIDestroy(&vhash));
4736     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
4737   }
4738   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
4739   /* Create cones */
4740   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
4741   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
4742   PetscCall(DMSetUp(dm));
4743   PetscCall(DMPlexGetCones(dm, &cones));
4744   for (c = 0; c < numCells; ++c) {
4745     for (p = 0; p < numCorners; ++p) {
4746       const PetscInt gv = cells[c * numCorners + p];
4747       PetscInt       lv;
4748 
4749       /* Positions within verticesAdj form 0-based local vertex numbering;
4750          we need to shift it by numCells to get correct DAG points (cells go first) */
4751       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
4752       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
4753       cones[c * numCorners + p] = lv + numCells;
4754     }
4755   }
4756   /* Build point sf */
4757   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
4758   PetscCall(PetscLayoutSetSize(layout, NVertices));
4759   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
4760   PetscCall(PetscLayoutSetBlockSize(layout, 1));
4761   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
4762   PetscCall(PetscLayoutDestroy(&layout));
4763   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
4764   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
4765   if (dm->sf) {
4766     const char *prefix;
4767 
4768     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
4769     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
4770   }
4771   PetscCall(DMSetPointSF(dm, sfPoint));
4772   PetscCall(PetscSFDestroy(&sfPoint));
4773   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)(*vertexSF), "Vertex Ownership SF"));
4774   /* Fill in the rest of the topology structure */
4775   PetscCall(DMPlexSymmetrize(dm));
4776   PetscCall(DMPlexStratify(dm));
4777   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4778   PetscFunctionReturn(0);
4779 }
4780 
4781 /*@C
4782   DMPlexBuildCoordinatesFromCellListParallel - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4783 
4784   Collective on dm
4785 
4786   Input Parameters:
4787 + dm - The `DM`
4788 . spaceDim - The spatial dimension used for coordinates
4789 . sfVert - `PetscSF` describing complete vertex ownership
4790 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4791 
4792   Level: advanced
4793 
4794   Fortran Note:
4795   Not currently supported in Fortran.
4796 
4797 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellListParallel()`
4798 @*/
4799 PetscErrorCode DMPlexBuildCoordinatesFromCellListParallel(DM dm, PetscInt spaceDim, PetscSF sfVert, const PetscReal vertexCoords[])
4800 {
4801   PetscSection coordSection;
4802   Vec          coordinates;
4803   PetscScalar *coords;
4804   PetscInt     numVertices, numVerticesAdj, coordSize, v, vStart, vEnd;
4805 
4806   PetscFunctionBegin;
4807   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4808   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
4809   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4810   PetscCall(DMSetCoordinateDim(dm, spaceDim));
4811   PetscCall(PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL));
4812   PetscCheck(vEnd - vStart == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Supplied sfVert has wrong number of leaves = %" PetscInt_FMT " != %" PetscInt_FMT " = vEnd - vStart", numVerticesAdj, vEnd - vStart);
4813   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4814   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4815   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
4816   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
4817   for (v = vStart; v < vEnd; ++v) {
4818     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
4819     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
4820   }
4821   PetscCall(PetscSectionSetUp(coordSection));
4822   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4823   PetscCall(VecCreate(PetscObjectComm((PetscObject)dm), &coordinates));
4824   PetscCall(VecSetBlockSize(coordinates, spaceDim));
4825   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4826   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4827   PetscCall(VecSetType(coordinates, VECSTANDARD));
4828   PetscCall(VecGetArray(coordinates, &coords));
4829   {
4830     MPI_Datatype coordtype;
4831 
4832     /* Need a temp buffer for coords if we have complex/single */
4833     PetscCallMPI(MPI_Type_contiguous(spaceDim, MPIU_SCALAR, &coordtype));
4834     PetscCallMPI(MPI_Type_commit(&coordtype));
4835 #if defined(PETSC_USE_COMPLEX)
4836     {
4837       PetscScalar *svertexCoords;
4838       PetscInt     i;
4839       PetscCall(PetscMalloc1(numVertices * spaceDim, &svertexCoords));
4840       for (i = 0; i < numVertices * spaceDim; i++) svertexCoords[i] = vertexCoords[i];
4841       PetscCall(PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4842       PetscCall(PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4843       PetscCall(PetscFree(svertexCoords));
4844     }
4845 #else
4846     PetscCall(PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4847     PetscCall(PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4848 #endif
4849     PetscCallMPI(MPI_Type_free(&coordtype));
4850   }
4851   PetscCall(VecRestoreArray(coordinates, &coords));
4852   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4853   PetscCall(VecDestroy(&coordinates));
4854   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4855   PetscFunctionReturn(0);
4856 }
4857 
4858 /*@
4859   DMPlexCreateFromCellListParallelPetsc - Create distributed `DMPLEX` from a list of vertices for each cell (common mesh generator output)
4860 
4861   Collective
4862 
4863   Input Parameters:
4864 + comm - The communicator
4865 . dim - The topological dimension of the mesh
4866 . numCells - The number of cells owned by this process
4867 . numVertices - The number of vertices owned by this process, or `PETSC_DECIDE`
4868 . NVertices - The global number of vertices, or `PETSC_DECIDE`
4869 . numCorners - The number of vertices for each cell
4870 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4871 . cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4872 . spaceDim - The spatial dimension used for coordinates
4873 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4874 
4875   Output Parameters:
4876 + dm - The `DM`
4877 . vertexSF - (Optional) `PetscSF` describing complete vertex ownership
4878 - verticesAdjSaved - (Optional) vertex adjacency array
4879 
4880   Level: intermediate
4881 
4882   Notes:
4883   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`,
4884   `DMPlexBuildFromCellListParallel()`, `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellListParallel()`
4885 
4886   See `DMPlexBuildFromCellListParallel()` for an example and details about the topology-related parameters.
4887 
4888   See `DMPlexBuildCoordinatesFromCellListParallel()` for details about the geometry-related parameters.
4889 
4890 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
4891 @*/
4892 PetscErrorCode DMPlexCreateFromCellListParallelPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, PetscInt **verticesAdj, DM *dm)
4893 {
4894   PetscSF sfVert;
4895 
4896   PetscFunctionBegin;
4897   PetscCall(DMCreate(comm, dm));
4898   PetscCall(DMSetType(*dm, DMPLEX));
4899   PetscValidLogicalCollectiveInt(*dm, dim, 2);
4900   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
4901   PetscCall(DMSetDimension(*dm, dim));
4902   PetscCall(DMPlexBuildFromCellListParallel(*dm, numCells, numVertices, NVertices, numCorners, cells, &sfVert, verticesAdj));
4903   if (interpolate) {
4904     DM idm;
4905 
4906     PetscCall(DMPlexInterpolate(*dm, &idm));
4907     PetscCall(DMDestroy(dm));
4908     *dm = idm;
4909   }
4910   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
4911   if (vertexSF) *vertexSF = sfVert;
4912   else PetscCall(PetscSFDestroy(&sfVert));
4913   PetscFunctionReturn(0);
4914 }
4915 
4916 /*@C
4917   DMPlexBuildFromCellList - Build `DMPLEX` topology from a list of vertices for each cell (common mesh generator output)
4918 
4919   Collective on dm
4920 
4921   Input Parameters:
4922 + dm - The `DM`
4923 . numCells - The number of cells owned by this process
4924 . numVertices - The number of vertices owned by this process, or `PETSC_DETERMINE`
4925 . numCorners - The number of vertices for each cell
4926 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4927 
4928   Level: advanced
4929 
4930   Notes:
4931   Two triangles sharing a face
4932 .vb
4933 
4934         2
4935       / | \
4936      /  |  \
4937     /   |   \
4938    0  0 | 1  3
4939     \   |   /
4940      \  |  /
4941       \ | /
4942         1
4943 .ve
4944 would have input
4945 .vb
4946   numCells = 2, numVertices = 4
4947   cells = [0 1 2  1 3 2]
4948 .ve
4949 which would result in the `DMPLEX`
4950 .vb
4951 
4952         4
4953       / | \
4954      /  |  \
4955     /   |   \
4956    2  0 | 1  5
4957     \   |   /
4958      \  |  /
4959       \ | /
4960         3
4961 .ve
4962 
4963   If numVertices is `PETSC_DETERMINE`, it is computed by PETSc as the maximum vertex index in cells + 1.
4964 
4965   Not currently supported in Fortran.
4966 
4967 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListPetsc()`
4968 @*/
4969 PetscErrorCode DMPlexBuildFromCellList(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const PetscInt cells[])
4970 {
4971   PetscInt *cones, c, p, dim;
4972 
4973   PetscFunctionBegin;
4974   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4975   PetscCall(DMGetDimension(dm, &dim));
4976   /* Get/check global number of vertices */
4977   {
4978     PetscInt       NVerticesInCells, i;
4979     const PetscInt len = numCells * numCorners;
4980 
4981     /* NVerticesInCells = max(cells) + 1 */
4982     NVerticesInCells = PETSC_MIN_INT;
4983     for (i = 0; i < len; i++)
4984       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4985     ++NVerticesInCells;
4986 
4987     if (numVertices == PETSC_DECIDE) numVertices = NVerticesInCells;
4988     else
4989       PetscCheck(numVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, numVertices, NVerticesInCells);
4990   }
4991   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
4992   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
4993   PetscCall(DMSetUp(dm));
4994   PetscCall(DMPlexGetCones(dm, &cones));
4995   for (c = 0; c < numCells; ++c) {
4996     for (p = 0; p < numCorners; ++p) cones[c * numCorners + p] = cells[c * numCorners + p] + numCells;
4997   }
4998   PetscCall(DMPlexSymmetrize(dm));
4999   PetscCall(DMPlexStratify(dm));
5000   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5001   PetscFunctionReturn(0);
5002 }
5003 
5004 /*@C
5005   DMPlexBuildCoordinatesFromCellList - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
5006 
5007   Collective on dm
5008 
5009   Input Parameters:
5010 + dm - The `DM`
5011 . spaceDim - The spatial dimension used for coordinates
5012 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
5013 
5014   Level: advanced
5015 
5016   Fortran Note:
5017   Not currently supported in Fortran.
5018 
5019 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellList()`
5020 @*/
5021 PetscErrorCode DMPlexBuildCoordinatesFromCellList(DM dm, PetscInt spaceDim, const PetscReal vertexCoords[])
5022 {
5023   PetscSection coordSection;
5024   Vec          coordinates;
5025   DM           cdm;
5026   PetscScalar *coords;
5027   PetscInt     v, vStart, vEnd, d;
5028 
5029   PetscFunctionBegin;
5030   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
5031   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
5032   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
5033   PetscCall(DMSetCoordinateDim(dm, spaceDim));
5034   PetscCall(DMGetCoordinateSection(dm, &coordSection));
5035   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5036   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
5037   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
5038   for (v = vStart; v < vEnd; ++v) {
5039     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
5040     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
5041   }
5042   PetscCall(PetscSectionSetUp(coordSection));
5043 
5044   PetscCall(DMGetCoordinateDM(dm, &cdm));
5045   PetscCall(DMCreateLocalVector(cdm, &coordinates));
5046   PetscCall(VecSetBlockSize(coordinates, spaceDim));
5047   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5048   PetscCall(VecGetArrayWrite(coordinates, &coords));
5049   for (v = 0; v < vEnd - vStart; ++v) {
5050     for (d = 0; d < spaceDim; ++d) coords[v * spaceDim + d] = vertexCoords[v * spaceDim + d];
5051   }
5052   PetscCall(VecRestoreArrayWrite(coordinates, &coords));
5053   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
5054   PetscCall(VecDestroy(&coordinates));
5055   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
5056   PetscFunctionReturn(0);
5057 }
5058 
5059 /*@
5060   DMPlexCreateFromCellListPetsc - Create `DMPLEX` from a list of vertices for each cell (common mesh generator output), but only process 0 takes in the input
5061 
5062   Collective
5063 
5064   Input Parameters:
5065 + comm - The communicator
5066 . dim - The topological dimension of the mesh
5067 . numCells - The number of cells, only on process 0
5068 . numVertices - The number of vertices owned by this process, or `PETSC_DECIDE`, only on process 0
5069 . numCorners - The number of vertices for each cell, only on process 0
5070 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
5071 . cells - An array of numCells*numCorners numbers, the vertices for each cell, only on process 0
5072 . spaceDim - The spatial dimension used for coordinates
5073 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex, only on process 0
5074 
5075   Output Parameter:
5076 . dm - The `DM`, which only has points on process 0
5077 
5078   Level: intermediate
5079 
5080   Notes:
5081   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`, `DMPlexBuildFromCellList()`,
5082   `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellList()`
5083 
5084   See `DMPlexBuildFromCellList()` for an example and details about the topology-related parameters.
5085   See `DMPlexBuildCoordinatesFromCellList()` for details about the geometry-related parameters.
5086   See `DMPlexCreateFromCellListParallelPetsc()` for parallel input
5087 
5088 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellList()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
5089 @*/
5090 PetscErrorCode DMPlexCreateFromCellListPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], DM *dm)
5091 {
5092   PetscMPIInt rank;
5093 
5094   PetscFunctionBegin;
5095   PetscCheck(dim, comm, PETSC_ERR_ARG_OUTOFRANGE, "This is not appropriate for 0-dimensional meshes. Consider either creating the DM using DMPlexCreateFromDAG(), by hand, or using DMSwarm.");
5096   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5097   PetscCall(DMCreate(comm, dm));
5098   PetscCall(DMSetType(*dm, DMPLEX));
5099   PetscCall(DMSetDimension(*dm, dim));
5100   if (rank == 0) PetscCall(DMPlexBuildFromCellList(*dm, numCells, numVertices, numCorners, cells));
5101   else PetscCall(DMPlexBuildFromCellList(*dm, 0, 0, 0, NULL));
5102   if (interpolate) {
5103     DM idm;
5104 
5105     PetscCall(DMPlexInterpolate(*dm, &idm));
5106     PetscCall(DMDestroy(dm));
5107     *dm = idm;
5108   }
5109   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, vertexCoords));
5110   else PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, NULL));
5111   PetscFunctionReturn(0);
5112 }
5113 
5114 /*@
5115   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a DM
5116 
5117   Input Parameters:
5118 + dm - The empty DM object, usually from DMCreate() and DMSetDimension()
5119 . depth - The depth of the DAG
5120 . numPoints - Array of size depth + 1 containing the number of points at each depth
5121 . coneSize - The cone size of each point
5122 . cones - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
5123 . coneOrientations - The orientation of each cone point
5124 - vertexCoords - An array of numPoints[0]*spacedim numbers representing the coordinates of each vertex, with spacedim the value set via DMSetCoordinateDim()
5125 
5126   Output Parameter:
5127 . dm - The DM
5128 
5129   Note:
5130   Two triangles sharing a face would have input
5131 .vb
5132   depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
5133   cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
5134  vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
5135 .ve
5136 which would result in the DMPlex
5137 .vb
5138         4
5139       / | \
5140      /  |  \
5141     /   |   \
5142    2  0 | 1  5
5143     \   |   /
5144      \  |  /
5145       \ | /
5146         3
5147 .ve
5148  Notice that all points are numbered consecutively, unlike `DMPlexCreateFromCellListPetsc()`
5149 
5150   Level: advanced
5151 
5152 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
5153 @*/
5154 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
5155 {
5156   Vec          coordinates;
5157   PetscSection coordSection;
5158   PetscScalar *coords;
5159   PetscInt     coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
5160 
5161   PetscFunctionBegin;
5162   PetscCall(DMGetDimension(dm, &dim));
5163   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
5164   PetscCheck(dimEmbed >= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Embedding dimension %" PetscInt_FMT " cannot be less than intrinsic dimension %" PetscInt_FMT, dimEmbed, dim);
5165   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
5166   PetscCall(DMPlexSetChart(dm, pStart, pEnd));
5167   for (p = pStart; p < pEnd; ++p) {
5168     PetscCall(DMPlexSetConeSize(dm, p, coneSize[p - pStart]));
5169     if (firstVertex < 0 && !coneSize[p - pStart]) firstVertex = p - pStart;
5170   }
5171   PetscCheck(firstVertex >= 0 || !numPoints[0], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Expected %" PetscInt_FMT " vertices but could not find any", numPoints[0]);
5172   PetscCall(DMSetUp(dm)); /* Allocate space for cones */
5173   for (p = pStart, off = 0; p < pEnd; off += coneSize[p - pStart], ++p) {
5174     PetscCall(DMPlexSetCone(dm, p, &cones[off]));
5175     PetscCall(DMPlexSetConeOrientation(dm, p, &coneOrientations[off]));
5176   }
5177   PetscCall(DMPlexSymmetrize(dm));
5178   PetscCall(DMPlexStratify(dm));
5179   /* Build coordinates */
5180   PetscCall(DMGetCoordinateSection(dm, &coordSection));
5181   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5182   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dimEmbed));
5183   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numPoints[0]));
5184   for (v = firstVertex; v < firstVertex + numPoints[0]; ++v) {
5185     PetscCall(PetscSectionSetDof(coordSection, v, dimEmbed));
5186     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed));
5187   }
5188   PetscCall(PetscSectionSetUp(coordSection));
5189   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
5190   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
5191   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5192   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
5193   PetscCall(VecSetBlockSize(coordinates, dimEmbed));
5194   PetscCall(VecSetType(coordinates, VECSTANDARD));
5195   if (vertexCoords) {
5196     PetscCall(VecGetArray(coordinates, &coords));
5197     for (v = 0; v < numPoints[0]; ++v) {
5198       PetscInt off;
5199 
5200       PetscCall(PetscSectionGetOffset(coordSection, v + firstVertex, &off));
5201       for (d = 0; d < dimEmbed; ++d) coords[off + d] = vertexCoords[v * dimEmbed + d];
5202     }
5203   }
5204   PetscCall(VecRestoreArray(coordinates, &coords));
5205   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
5206   PetscCall(VecDestroy(&coordinates));
5207   PetscFunctionReturn(0);
5208 }
5209 
5210 /*@C
5211   DMPlexCreateCellVertexFromFile - Create a `DMPLEX` mesh from a simple cell-vertex file.
5212 
5213   Collective
5214 
5215 + comm        - The MPI communicator
5216 . filename    - Name of the .dat file
5217 - interpolate - Create faces and edges in the mesh
5218 
5219   Output Parameter:
5220 . dm  - The `DM` object representing the mesh
5221 
5222   Level: beginner
5223 
5224   Note:
5225   The format is the simplest possible:
5226 .vb
5227   Ne
5228   v0 v1 ... vk
5229   Nv
5230   x y z marker
5231 .ve
5232 
5233   Developer Note:
5234   Should use a `PetscViewer` not a filename
5235 
5236 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromFile()`, `DMPlexCreateMedFromFile()`, `DMPlexCreateGmsh()`, `DMPlexCreate()`
5237 @*/
5238 PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
5239 {
5240   DMLabel      marker;
5241   PetscViewer  viewer;
5242   Vec          coordinates;
5243   PetscSection coordSection;
5244   PetscScalar *coords;
5245   char         line[PETSC_MAX_PATH_LEN];
5246   PetscInt     dim = 3, cdim = 3, coordSize, v, c, d;
5247   PetscMPIInt  rank;
5248   int          snum, Nv, Nc, Ncn, Nl;
5249 
5250   PetscFunctionBegin;
5251   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5252   PetscCall(PetscViewerCreate(comm, &viewer));
5253   PetscCall(PetscViewerSetType(viewer, PETSCVIEWERASCII));
5254   PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
5255   PetscCall(PetscViewerFileSetName(viewer, filename));
5256   if (rank == 0) {
5257     PetscCall(PetscViewerRead(viewer, line, 4, NULL, PETSC_STRING));
5258     snum = sscanf(line, "%d %d %d %d", &Nc, &Nv, &Ncn, &Nl);
5259     PetscCheck(snum == 4, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5260   } else {
5261     Nc = Nv = Ncn = Nl = 0;
5262   }
5263   PetscCall(DMCreate(comm, dm));
5264   PetscCall(DMSetType(*dm, DMPLEX));
5265   PetscCall(DMPlexSetChart(*dm, 0, Nc + Nv));
5266   PetscCall(DMSetDimension(*dm, dim));
5267   PetscCall(DMSetCoordinateDim(*dm, cdim));
5268   /* Read topology */
5269   if (rank == 0) {
5270     char     format[PETSC_MAX_PATH_LEN];
5271     PetscInt cone[8];
5272     int      vbuf[8], v;
5273 
5274     for (c = 0; c < Ncn; ++c) {
5275       format[c * 3 + 0] = '%';
5276       format[c * 3 + 1] = 'd';
5277       format[c * 3 + 2] = ' ';
5278     }
5279     format[Ncn * 3 - 1] = '\0';
5280     for (c = 0; c < Nc; ++c) PetscCall(DMPlexSetConeSize(*dm, c, Ncn));
5281     PetscCall(DMSetUp(*dm));
5282     for (c = 0; c < Nc; ++c) {
5283       PetscCall(PetscViewerRead(viewer, line, Ncn, NULL, PETSC_STRING));
5284       switch (Ncn) {
5285       case 2:
5286         snum = sscanf(line, format, &vbuf[0], &vbuf[1]);
5287         break;
5288       case 3:
5289         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2]);
5290         break;
5291       case 4:
5292         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3]);
5293         break;
5294       case 6:
5295         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5]);
5296         break;
5297       case 8:
5298         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);
5299         break;
5300       default:
5301         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No cell shape with %d vertices", Ncn);
5302       }
5303       PetscCheck(snum == Ncn, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5304       for (v = 0; v < Ncn; ++v) cone[v] = vbuf[v] + Nc;
5305       /* Hexahedra are inverted */
5306       if (Ncn == 8) {
5307         PetscInt tmp = cone[1];
5308         cone[1]      = cone[3];
5309         cone[3]      = tmp;
5310       }
5311       PetscCall(DMPlexSetCone(*dm, c, cone));
5312     }
5313   }
5314   PetscCall(DMPlexSymmetrize(*dm));
5315   PetscCall(DMPlexStratify(*dm));
5316   /* Read coordinates */
5317   PetscCall(DMGetCoordinateSection(*dm, &coordSection));
5318   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5319   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
5320   PetscCall(PetscSectionSetChart(coordSection, Nc, Nc + Nv));
5321   for (v = Nc; v < Nc + Nv; ++v) {
5322     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
5323     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
5324   }
5325   PetscCall(PetscSectionSetUp(coordSection));
5326   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
5327   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
5328   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5329   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
5330   PetscCall(VecSetBlockSize(coordinates, cdim));
5331   PetscCall(VecSetType(coordinates, VECSTANDARD));
5332   PetscCall(VecGetArray(coordinates, &coords));
5333   if (rank == 0) {
5334     char   format[PETSC_MAX_PATH_LEN];
5335     double x[3];
5336     int    l, val[3];
5337 
5338     if (Nl) {
5339       for (l = 0; l < Nl; ++l) {
5340         format[l * 3 + 0] = '%';
5341         format[l * 3 + 1] = 'd';
5342         format[l * 3 + 2] = ' ';
5343       }
5344       format[Nl * 3 - 1] = '\0';
5345       PetscCall(DMCreateLabel(*dm, "marker"));
5346       PetscCall(DMGetLabel(*dm, "marker", &marker));
5347     }
5348     for (v = 0; v < Nv; ++v) {
5349       PetscCall(PetscViewerRead(viewer, line, 3 + Nl, NULL, PETSC_STRING));
5350       snum = sscanf(line, "%lg %lg %lg", &x[0], &x[1], &x[2]);
5351       PetscCheck(snum == 3, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5352       switch (Nl) {
5353       case 0:
5354         snum = 0;
5355         break;
5356       case 1:
5357         snum = sscanf(line, format, &val[0]);
5358         break;
5359       case 2:
5360         snum = sscanf(line, format, &val[0], &val[1]);
5361         break;
5362       case 3:
5363         snum = sscanf(line, format, &val[0], &val[1], &val[2]);
5364         break;
5365       default:
5366         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Request support for %d labels", Nl);
5367       }
5368       PetscCheck(snum == Nl, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5369       for (d = 0; d < cdim; ++d) coords[v * cdim + d] = x[d];
5370       for (l = 0; l < Nl; ++l) PetscCall(DMLabelSetValue(marker, v + Nc, val[l]));
5371     }
5372   }
5373   PetscCall(VecRestoreArray(coordinates, &coords));
5374   PetscCall(DMSetCoordinatesLocal(*dm, coordinates));
5375   PetscCall(VecDestroy(&coordinates));
5376   PetscCall(PetscViewerDestroy(&viewer));
5377   if (interpolate) {
5378     DM      idm;
5379     DMLabel bdlabel;
5380 
5381     PetscCall(DMPlexInterpolate(*dm, &idm));
5382     PetscCall(DMDestroy(dm));
5383     *dm = idm;
5384 
5385     if (!Nl) {
5386       PetscCall(DMCreateLabel(*dm, "marker"));
5387       PetscCall(DMGetLabel(*dm, "marker", &bdlabel));
5388       PetscCall(DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel));
5389       PetscCall(DMPlexLabelComplete(*dm, bdlabel));
5390     }
5391   }
5392   PetscFunctionReturn(0);
5393 }
5394 
5395 /*@C
5396   DMPlexCreateFromFile - This takes a filename and produces a `DM`
5397 
5398   Collective
5399 
5400   Input Parameters:
5401 + comm - The communicator
5402 . filename - A file name
5403 . plexname - The object name of the resulting `DM`, also used for intra-datafile lookup by some formats
5404 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
5405 
5406   Output Parameter:
5407 . dm - The `DM`
5408 
5409   Options Database Key:
5410 . -dm_plex_create_from_hdf5_xdmf - use the `PETSC_VIEWER_HDF5_XDMF` format for reading HDF5
5411 
5412   Use -dm_plex_create_ prefix to pass options to the internal PetscViewer, e.g.
5413 $ -dm_plex_create_viewer_hdf5_collective
5414 
5415   Level: beginner
5416 
5417   Notes:
5418   Using `PETSCVIEWERHDF5` type with `PETSC_VIEWER_HDF5_PETSC` format, one can save multiple `DMPLEX`
5419   meshes in a single HDF5 file. This in turn requires one to name the `DMPLEX` object with `PetscObjectSetName()`
5420   before saving it with `DMView()` and before loading it with `DMLoad()` for identification of the mesh object.
5421   The input parameter name is thus used to name the `DMPLEX` object when `DMPlexCreateFromFile()` internally
5422   calls `DMLoad()`. Currently, name is ignored for other viewer types and/or formats.
5423 
5424 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`, `PetscObjectSetName()`, `DMView()`, `DMLoad()`
5425 @*/
5426 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], const char plexname[], PetscBool interpolate, DM *dm)
5427 {
5428   const char  extGmsh[]      = ".msh";
5429   const char  extGmsh2[]     = ".msh2";
5430   const char  extGmsh4[]     = ".msh4";
5431   const char  extCGNS[]      = ".cgns";
5432   const char  extExodus[]    = ".exo";
5433   const char  extExodus_e[]  = ".e";
5434   const char  extGenesis[]   = ".gen";
5435   const char  extFluent[]    = ".cas";
5436   const char  extHDF5[]      = ".h5";
5437   const char  extMed[]       = ".med";
5438   const char  extPLY[]       = ".ply";
5439   const char  extEGADSLite[] = ".egadslite";
5440   const char  extEGADS[]     = ".egads";
5441   const char  extIGES[]      = ".igs";
5442   const char  extSTEP[]      = ".stp";
5443   const char  extCV[]        = ".dat";
5444   size_t      len;
5445   PetscBool   isGmsh, isGmsh2, isGmsh4, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isMed, isPLY, isEGADSLite, isEGADS, isIGES, isSTEP, isCV;
5446   PetscMPIInt rank;
5447 
5448   PetscFunctionBegin;
5449   PetscValidCharPointer(filename, 2);
5450   if (plexname) PetscValidCharPointer(plexname, 3);
5451   PetscValidPointer(dm, 5);
5452   PetscCall(DMInitializePackage());
5453   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5454   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5455   PetscCall(PetscStrlen(filename, &len));
5456   PetscCheck(len, comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
5457 
5458 #define CheckExtension(extension__, is_extension__) \
5459   do { \
5460     PetscAssert(sizeof(extension__), comm, PETSC_ERR_PLIB, "Zero-size extension: %s", extension__); \
5461     /* don't count the null-terminator at the end */ \
5462     const size_t ext_len = sizeof(extension__) - 1; \
5463     if (len < ext_len) { \
5464       is_extension__ = PETSC_FALSE; \
5465     } else { \
5466       PetscCall(PetscStrncmp(filename + len - ext_len, extension__, ext_len, &is_extension__)); \
5467     } \
5468   } while (0)
5469 
5470   CheckExtension(extGmsh, isGmsh);
5471   CheckExtension(extGmsh2, isGmsh2);
5472   CheckExtension(extGmsh4, isGmsh4);
5473   CheckExtension(extCGNS, isCGNS);
5474   CheckExtension(extExodus, isExodus);
5475   if (!isExodus) CheckExtension(extExodus_e, isExodus);
5476   CheckExtension(extGenesis, isGenesis);
5477   CheckExtension(extFluent, isFluent);
5478   CheckExtension(extHDF5, isHDF5);
5479   CheckExtension(extMed, isMed);
5480   CheckExtension(extPLY, isPLY);
5481   CheckExtension(extEGADSLite, isEGADSLite);
5482   CheckExtension(extEGADS, isEGADS);
5483   CheckExtension(extIGES, isIGES);
5484   CheckExtension(extSTEP, isSTEP);
5485   CheckExtension(extCV, isCV);
5486 
5487 #undef CheckExtension
5488 
5489   if (isGmsh || isGmsh2 || isGmsh4) {
5490     PetscCall(DMPlexCreateGmshFromFile(comm, filename, interpolate, dm));
5491   } else if (isCGNS) {
5492     PetscCall(DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm));
5493   } else if (isExodus || isGenesis) {
5494     PetscCall(DMPlexCreateExodusFromFile(comm, filename, interpolate, dm));
5495   } else if (isFluent) {
5496     PetscCall(DMPlexCreateFluentFromFile(comm, filename, interpolate, dm));
5497   } else if (isHDF5) {
5498     PetscBool   load_hdf5_xdmf = PETSC_FALSE;
5499     PetscViewer viewer;
5500 
5501     /* PETSC_VIEWER_HDF5_XDMF is used if the filename ends with .xdmf.h5, or if -dm_plex_create_from_hdf5_xdmf option is present */
5502     PetscCall(PetscStrncmp(&filename[PetscMax(0, len - 8)], ".xdmf", 5, &load_hdf5_xdmf));
5503     PetscCall(PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &load_hdf5_xdmf, NULL));
5504     PetscCall(PetscViewerCreate(comm, &viewer));
5505     PetscCall(PetscViewerSetType(viewer, PETSCVIEWERHDF5));
5506     PetscCall(PetscViewerSetOptionsPrefix(viewer, "dm_plex_create_"));
5507     PetscCall(PetscViewerSetFromOptions(viewer));
5508     PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
5509     PetscCall(PetscViewerFileSetName(viewer, filename));
5510 
5511     PetscCall(DMCreate(comm, dm));
5512     PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5513     PetscCall(DMSetType(*dm, DMPLEX));
5514     if (load_hdf5_xdmf) PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF));
5515     PetscCall(DMLoad(*dm, viewer));
5516     if (load_hdf5_xdmf) PetscCall(PetscViewerPopFormat(viewer));
5517     PetscCall(PetscViewerDestroy(&viewer));
5518 
5519     if (interpolate) {
5520       DM idm;
5521 
5522       PetscCall(DMPlexInterpolate(*dm, &idm));
5523       PetscCall(DMDestroy(dm));
5524       *dm = idm;
5525     }
5526   } else if (isMed) {
5527     PetscCall(DMPlexCreateMedFromFile(comm, filename, interpolate, dm));
5528   } else if (isPLY) {
5529     PetscCall(DMPlexCreatePLYFromFile(comm, filename, interpolate, dm));
5530   } else if (isEGADSLite || isEGADS || isIGES || isSTEP) {
5531     if (isEGADSLite) PetscCall(DMPlexCreateEGADSLiteFromFile(comm, filename, dm));
5532     else PetscCall(DMPlexCreateEGADSFromFile(comm, filename, dm));
5533     if (!interpolate) {
5534       DM udm;
5535 
5536       PetscCall(DMPlexUninterpolate(*dm, &udm));
5537       PetscCall(DMDestroy(dm));
5538       *dm = udm;
5539     }
5540   } else if (isCV) {
5541     PetscCall(DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm));
5542   } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
5543   PetscCall(PetscStrlen(plexname, &len));
5544   if (len) PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5545   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5546   PetscFunctionReturn(0);
5547 }
5548 /*@C
5549   DMPlexCreateEphemeral - This takes a `DMPlexTransform` and a base `DMPlex` and produces an ephemeral `DM`, meaning one that is created on the fly in response to queries.
5550 
5551   Input Parameter:
5552 . tr - The `DMPlexTransform`
5553 
5554   Output Parameter:
5555 . dm - The `DM`
5556 
5557   Notes:
5558   An emphemeral mesh is one that is not stored concretely, as in the default Plex implementation, but rather is produced on the fly in response to queries, using information from the transform and the base mesh.
5559 
5560   Level: beginner
5561 
5562 .seealso: `DMPlexCreateFromFile`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
5563 @*/
5564 PetscErrorCode DMPlexCreateEphemeral(DMPlexTransform tr, DM *dm)
5565 {
5566   DM       bdm;
5567   PetscInt Nl;
5568 
5569   PetscFunctionBegin;
5570   PetscCall(DMCreate(PetscObjectComm((PetscObject)tr), dm));
5571   PetscCall(DMSetType(*dm, DMPLEX));
5572   PetscCall(DMSetFromOptions(*dm));
5573 
5574   PetscCall(PetscObjectReference((PetscObject)tr));
5575   PetscCall(DMPlexTransformDestroy(&((DM_Plex *)(*dm)->data)->tr));
5576   ((DM_Plex *)(*dm)->data)->tr = tr;
5577 
5578   PetscCall(DMPlexTransformGetDM(tr, &bdm));
5579   PetscCall(DMGetNumLabels(bdm, &Nl));
5580   for (PetscInt l = 0; l < Nl; ++l) {
5581     DMLabel     label, labelNew;
5582     const char *lname;
5583     PetscBool   isDepth, isCellType;
5584 
5585     PetscCall(DMGetLabelName(bdm, l, &lname));
5586     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
5587     if (isDepth) continue;
5588     PetscCall(PetscStrcmp(lname, "celltype", &isCellType));
5589     if (isCellType) continue;
5590     PetscCall(DMCreateLabel(*dm, lname));
5591     PetscCall(DMGetLabel(bdm, lname, &label));
5592     PetscCall(DMGetLabel(*dm, lname, &labelNew));
5593     PetscCall(DMLabelSetType(labelNew, DMLABELEPHEMERAL));
5594     PetscCall(DMLabelEphemeralSetLabel(labelNew, label));
5595     PetscCall(DMLabelEphemeralSetTransform(labelNew, tr));
5596     PetscCall(DMLabelSetUp(labelNew));
5597   }
5598   PetscFunctionReturn(0);
5599 }
5600