xref: /petsc/src/dm/impls/plex/plexcreate.c (revision e44f6aeb0c7eee2716d50e70cd976c48a5d3c225)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h> /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>   /*I   "petscdmplex.h"   I*/
4 #include <petscsf.h>
5 #include <petscdmplextransform.h>
6 #include <petscdmlabelephemeral.h>
7 #include <petsc/private/kernels/blockmatmult.h>
8 #include <petsc/private/kernels/blockinvert.h>
9 
10 PetscLogEvent DMPLEX_CreateFromFile, DMPLEX_CreateFromOptions, DMPLEX_BuildFromCellList, DMPLEX_BuildCoordinatesFromCellList;
11 
12 /* External function declarations here */
13 static PetscErrorCode DMInitialize_Plex(DM dm);
14 
15 /* This copies internal things in the Plex structure that we generally want when making a new, related Plex */
16 PetscErrorCode DMPlexCopy_Internal(DM dmin, PetscBool copyPeriodicity, PetscBool copyOverlap, DM dmout)
17 {
18   const PetscReal         *maxCell, *Lstart, *L;
19   PetscBool                dist;
20   DMPlexReorderDefaultFlag reorder;
21 
22   PetscFunctionBegin;
23   if (copyPeriodicity) {
24     PetscCall(DMGetPeriodicity(dmin, &maxCell, &Lstart, &L));
25     PetscCall(DMSetPeriodicity(dmout, maxCell, Lstart, L));
26   }
27   PetscCall(DMPlexDistributeGetDefault(dmin, &dist));
28   PetscCall(DMPlexDistributeSetDefault(dmout, dist));
29   PetscCall(DMPlexReorderGetDefault(dmin, &reorder));
30   PetscCall(DMPlexReorderSetDefault(dmout, reorder));
31   ((DM_Plex *)dmout->data)->useHashLocation = ((DM_Plex *)dmin->data)->useHashLocation;
32   if (copyOverlap) PetscCall(DMPlexSetOverlap_Plex(dmout, dmin, 0));
33   PetscFunctionReturn(PETSC_SUCCESS);
34 }
35 
36 /* Replace dm with the contents of ndm, and then destroy ndm
37    - Share the DM_Plex structure
38    - Share the coordinates
39    - Share the SF
40 */
41 PetscErrorCode DMPlexReplace_Internal(DM dm, DM *ndm)
42 {
43   PetscSF          sf;
44   DM               dmNew = *ndm, coordDM, coarseDM;
45   Vec              coords;
46   const PetscReal *maxCell, *Lstart, *L;
47   PetscInt         dim, cdim;
48 
49   PetscFunctionBegin;
50   if (dm == dmNew) {
51     PetscCall(DMDestroy(ndm));
52     PetscFunctionReturn(PETSC_SUCCESS);
53   }
54   dm->setupcalled = dmNew->setupcalled;
55   PetscCall(DMGetDimension(dmNew, &dim));
56   PetscCall(DMSetDimension(dm, dim));
57   PetscCall(DMGetCoordinateDim(dmNew, &cdim));
58   PetscCall(DMSetCoordinateDim(dm, cdim));
59   PetscCall(DMGetPointSF(dmNew, &sf));
60   PetscCall(DMSetPointSF(dm, sf));
61   PetscCall(DMGetCoordinateDM(dmNew, &coordDM));
62   PetscCall(DMGetCoordinatesLocal(dmNew, &coords));
63   PetscCall(DMSetCoordinateDM(dm, coordDM));
64   PetscCall(DMSetCoordinatesLocal(dm, coords));
65   PetscCall(DMGetCellCoordinateDM(dmNew, &coordDM));
66   PetscCall(DMGetCellCoordinatesLocal(dmNew, &coords));
67   PetscCall(DMSetCellCoordinateDM(dm, coordDM));
68   PetscCall(DMSetCellCoordinatesLocal(dm, coords));
69   /* Do not want to create the coordinate field if it does not already exist, so do not call DMGetCoordinateField() */
70   PetscCall(DMFieldDestroy(&dm->coordinates[0].field));
71   dm->coordinates[0].field            = dmNew->coordinates[0].field;
72   ((DM_Plex *)dmNew->data)->coordFunc = ((DM_Plex *)dm->data)->coordFunc;
73   PetscCall(DMGetPeriodicity(dmNew, &maxCell, &Lstart, &L));
74   PetscCall(DMSetPeriodicity(dm, maxCell, Lstart, L));
75   PetscCall(DMDestroy_Plex(dm));
76   PetscCall(DMInitialize_Plex(dm));
77   dm->data = dmNew->data;
78   ((DM_Plex *)dmNew->data)->refct++;
79   PetscCall(DMPlexGetIsoperiodicFaceSF(dm, &sf));
80   PetscCall(DMPlexSetIsoperiodicFaceSF(dm, sf)); // for the compose function effect on dm
81   PetscCall(DMDestroyLabelLinkList_Internal(dm));
82   PetscCall(DMCopyLabels(dmNew, dm, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL));
83   PetscCall(DMGetCoarseDM(dmNew, &coarseDM));
84   PetscCall(DMSetCoarseDM(dm, coarseDM));
85   PetscCall(DMDestroy(ndm));
86   PetscFunctionReturn(PETSC_SUCCESS);
87 }
88 
89 /* Swap dm with the contents of dmNew
90    - Swap the DM_Plex structure
91    - Swap the coordinates
92    - Swap the point PetscSF
93 */
94 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
95 {
96   DM          coordDMA, coordDMB;
97   Vec         coordsA, coordsB;
98   PetscSF     sfA, sfB;
99   DMField     fieldTmp;
100   void       *tmp;
101   DMLabelLink listTmp;
102   DMLabel     depthTmp;
103   PetscInt    tmpI;
104 
105   PetscFunctionBegin;
106   if (dmA == dmB) PetscFunctionReturn(PETSC_SUCCESS);
107   PetscCall(DMGetPointSF(dmA, &sfA));
108   PetscCall(DMGetPointSF(dmB, &sfB));
109   PetscCall(PetscObjectReference((PetscObject)sfA));
110   PetscCall(DMSetPointSF(dmA, sfB));
111   PetscCall(DMSetPointSF(dmB, sfA));
112   PetscCall(PetscObjectDereference((PetscObject)sfA));
113 
114   PetscCall(DMGetCoordinateDM(dmA, &coordDMA));
115   PetscCall(DMGetCoordinateDM(dmB, &coordDMB));
116   PetscCall(PetscObjectReference((PetscObject)coordDMA));
117   PetscCall(DMSetCoordinateDM(dmA, coordDMB));
118   PetscCall(DMSetCoordinateDM(dmB, coordDMA));
119   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
120 
121   PetscCall(DMGetCoordinatesLocal(dmA, &coordsA));
122   PetscCall(DMGetCoordinatesLocal(dmB, &coordsB));
123   PetscCall(PetscObjectReference((PetscObject)coordsA));
124   PetscCall(DMSetCoordinatesLocal(dmA, coordsB));
125   PetscCall(DMSetCoordinatesLocal(dmB, coordsA));
126   PetscCall(PetscObjectDereference((PetscObject)coordsA));
127 
128   PetscCall(DMGetCellCoordinateDM(dmA, &coordDMA));
129   PetscCall(DMGetCellCoordinateDM(dmB, &coordDMB));
130   PetscCall(PetscObjectReference((PetscObject)coordDMA));
131   PetscCall(DMSetCellCoordinateDM(dmA, coordDMB));
132   PetscCall(DMSetCellCoordinateDM(dmB, coordDMA));
133   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
134 
135   PetscCall(DMGetCellCoordinatesLocal(dmA, &coordsA));
136   PetscCall(DMGetCellCoordinatesLocal(dmB, &coordsB));
137   PetscCall(PetscObjectReference((PetscObject)coordsA));
138   PetscCall(DMSetCellCoordinatesLocal(dmA, coordsB));
139   PetscCall(DMSetCellCoordinatesLocal(dmB, coordsA));
140   PetscCall(PetscObjectDereference((PetscObject)coordsA));
141 
142   fieldTmp                  = dmA->coordinates[0].field;
143   dmA->coordinates[0].field = dmB->coordinates[0].field;
144   dmB->coordinates[0].field = fieldTmp;
145   fieldTmp                  = dmA->coordinates[1].field;
146   dmA->coordinates[1].field = dmB->coordinates[1].field;
147   dmB->coordinates[1].field = fieldTmp;
148   tmp                       = dmA->data;
149   dmA->data                 = dmB->data;
150   dmB->data                 = tmp;
151   listTmp                   = dmA->labels;
152   dmA->labels               = dmB->labels;
153   dmB->labels               = listTmp;
154   depthTmp                  = dmA->depthLabel;
155   dmA->depthLabel           = dmB->depthLabel;
156   dmB->depthLabel           = depthTmp;
157   depthTmp                  = dmA->celltypeLabel;
158   dmA->celltypeLabel        = dmB->celltypeLabel;
159   dmB->celltypeLabel        = depthTmp;
160   tmpI                      = dmA->levelup;
161   dmA->levelup              = dmB->levelup;
162   dmB->levelup              = tmpI;
163   PetscFunctionReturn(PETSC_SUCCESS);
164 }
165 
166 PetscErrorCode DMPlexInterpolateInPlace_Internal(DM dm)
167 {
168   DM idm;
169 
170   PetscFunctionBegin;
171   PetscCall(DMPlexInterpolate(dm, &idm));
172   PetscCall(DMPlexCopyCoordinates(dm, idm));
173   PetscCall(DMPlexReplace_Internal(dm, &idm));
174   PetscFunctionReturn(PETSC_SUCCESS);
175 }
176 
177 /*@C
178   DMPlexCreateCoordinateSpace - Creates a finite element space for the coordinates
179 
180   Collective
181 
182   Input Parameters:
183 + dm        - The `DMPLEX`
184 . degree    - The degree of the finite element or `PETSC_DECIDE`
185 . project   - Flag to project current coordinates into the space
186 - coordFunc - An optional function to map new points from refinement to the surface
187 
188   Level: advanced
189 
190 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `PetscPointFunc`, `PetscFECreateLagrange()`, `DMGetCoordinateDM()`
191 @*/
192 PetscErrorCode DMPlexCreateCoordinateSpace(DM dm, PetscInt degree, PetscBool project, PetscPointFunc coordFunc)
193 {
194   DM_Plex *mesh = (DM_Plex *)dm->data;
195   PetscFE  fe   = NULL;
196   DM       cdm;
197   PetscInt dim, dE, qorder;
198 
199   PetscFunctionBegin;
200   PetscCall(DMGetDimension(dm, &dim));
201   PetscCall(DMGetCoordinateDim(dm, &dE));
202   qorder = degree;
203   PetscCall(DMGetCoordinateDM(dm, &cdm));
204   PetscObjectOptionsBegin((PetscObject)cdm);
205   PetscCall(PetscOptionsBoundedInt("-default_quadrature_order", "Quadrature order is one less than quadrature points per edge", "DMPlexCreateCoordinateSpace", qorder, &qorder, NULL, 0));
206   PetscOptionsEnd();
207   if (degree >= 0) {
208     DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
209     PetscInt       cStart, cEnd, gct;
210 
211     PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
212     if (cEnd > cStart) PetscCall(DMPlexGetCellType(dm, cStart, &ct));
213     gct = (PetscInt)ct;
214     PetscCall(MPIU_Allreduce(MPI_IN_PLACE, &gct, 1, MPIU_INT, MPI_MIN, PetscObjectComm((PetscObject)dm)));
215     ct = (DMPolytopeType)gct;
216     // Work around current bug in PetscDualSpaceSetUp_Lagrange()
217     //   Can be seen in plex_tutorials-ex10_1
218     if (ct != DM_POLYTOPE_SEG_PRISM_TENSOR && ct != DM_POLYTOPE_TRI_PRISM_TENSOR && ct != DM_POLYTOPE_QUAD_PRISM_TENSOR) PetscCall(PetscFECreateLagrangeByCell(PETSC_COMM_SELF, dim, dE, ct, degree, qorder, &fe));
219   }
220   PetscCall(DMSetCoordinateDisc(dm, fe, project));
221   PetscCall(PetscFEDestroy(&fe));
222   mesh->coordFunc = coordFunc;
223   PetscFunctionReturn(PETSC_SUCCESS);
224 }
225 
226 /*@
227   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
228 
229   Collective
230 
231   Input Parameters:
232 + comm            - The communicator for the `DM` object
233 . dim             - The spatial dimension
234 . simplex         - Flag for simplicial cells, otherwise they are tensor product cells
235 . interpolate     - Flag to create intermediate mesh pieces (edges, faces)
236 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
237 
238   Output Parameter:
239 . newdm - The `DM` object
240 
241   Level: beginner
242 
243 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetType()`, `DMCreate()`
244 @*/
245 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscReal refinementLimit, DM *newdm)
246 {
247   DM          dm;
248   PetscMPIInt rank;
249 
250   PetscFunctionBegin;
251   PetscCall(DMCreate(comm, &dm));
252   PetscCall(DMSetType(dm, DMPLEX));
253   PetscCall(DMSetDimension(dm, dim));
254   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
255   PetscCallMPI(MPI_Comm_rank(comm, &rank));
256   switch (dim) {
257   case 2:
258     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "triangular"));
259     else PetscCall(PetscObjectSetName((PetscObject)dm, "quadrilateral"));
260     break;
261   case 3:
262     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "tetrahedral"));
263     else PetscCall(PetscObjectSetName((PetscObject)dm, "hexahedral"));
264     break;
265   default:
266     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
267   }
268   if (rank) {
269     PetscInt numPoints[2] = {0, 0};
270     PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL));
271   } else {
272     switch (dim) {
273     case 2:
274       if (simplex) {
275         PetscInt    numPoints[2]        = {4, 2};
276         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
277         PetscInt    cones[6]            = {2, 3, 4, 5, 4, 3};
278         PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
279         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
280 
281         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
282       } else {
283         PetscInt    numPoints[2]        = {6, 2};
284         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
285         PetscInt    cones[8]            = {2, 3, 4, 5, 3, 6, 7, 4};
286         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
287         PetscScalar vertexCoords[12]    = {-1.0, -0.5, 0.0, -0.5, 0.0, 0.5, -1.0, 0.5, 1.0, -0.5, 1.0, 0.5};
288 
289         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
290       }
291       break;
292     case 3:
293       if (simplex) {
294         PetscInt    numPoints[2]        = {5, 2};
295         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
296         PetscInt    cones[8]            = {4, 3, 5, 2, 5, 3, 4, 6};
297         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
298         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0};
299 
300         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
301       } else {
302         PetscInt    numPoints[2]         = {12, 2};
303         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
304         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9, 5, 4, 10, 11, 7, 12, 13, 8};
305         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
306         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5, -1.0, 0.5, -0.5, 0.0, 0.5, -0.5, 0.0, -0.5, -0.5, -1.0, -0.5, 0.5, 0.0, -0.5, 0.5, 0.0, 0.5, 0.5, -1.0, 0.5, 0.5, 1.0, 0.5, -0.5, 1.0, -0.5, -0.5, 1.0, -0.5, 0.5, 1.0, 0.5, 0.5};
307 
308         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
309       }
310       break;
311     default:
312       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
313     }
314   }
315   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
316   *newdm = dm;
317   if (refinementLimit > 0.0) {
318     DM          rdm;
319     const char *name;
320 
321     PetscCall(DMPlexSetRefinementUniform(*newdm, PETSC_FALSE));
322     PetscCall(DMPlexSetRefinementLimit(*newdm, refinementLimit));
323     PetscCall(DMRefine(*newdm, comm, &rdm));
324     PetscCall(PetscObjectGetName((PetscObject)*newdm, &name));
325     PetscCall(PetscObjectSetName((PetscObject)rdm, name));
326     PetscCall(DMDestroy(newdm));
327     *newdm = rdm;
328   }
329   if (interpolate) {
330     DM idm;
331 
332     PetscCall(DMPlexInterpolate(*newdm, &idm));
333     PetscCall(DMDestroy(newdm));
334     *newdm = idm;
335   }
336   PetscFunctionReturn(PETSC_SUCCESS);
337 }
338 
339 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
340 {
341   const PetscInt numVertices    = 2;
342   PetscInt       markerRight    = 1;
343   PetscInt       markerLeft     = 1;
344   PetscBool      markerSeparate = PETSC_FALSE;
345   Vec            coordinates;
346   PetscSection   coordSection;
347   PetscScalar   *coords;
348   PetscInt       coordSize;
349   PetscMPIInt    rank;
350   PetscInt       cdim = 1, v;
351 
352   PetscFunctionBegin;
353   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
354   if (markerSeparate) {
355     markerRight = 2;
356     markerLeft  = 1;
357   }
358   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
359   if (rank == 0) {
360     PetscCall(DMPlexSetChart(dm, 0, numVertices));
361     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
362     PetscCall(DMSetLabelValue(dm, "marker", 0, markerLeft));
363     PetscCall(DMSetLabelValue(dm, "marker", 1, markerRight));
364   }
365   PetscCall(DMPlexSymmetrize(dm));
366   PetscCall(DMPlexStratify(dm));
367   /* Build coordinates */
368   PetscCall(DMSetCoordinateDim(dm, cdim));
369   PetscCall(DMGetCoordinateSection(dm, &coordSection));
370   PetscCall(PetscSectionSetNumFields(coordSection, 1));
371   PetscCall(PetscSectionSetChart(coordSection, 0, numVertices));
372   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
373   for (v = 0; v < numVertices; ++v) {
374     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
375     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
376   }
377   PetscCall(PetscSectionSetUp(coordSection));
378   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
379   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
380   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
381   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
382   PetscCall(VecSetBlockSize(coordinates, cdim));
383   PetscCall(VecSetType(coordinates, VECSTANDARD));
384   PetscCall(VecGetArray(coordinates, &coords));
385   coords[0] = lower[0];
386   coords[1] = upper[0];
387   PetscCall(VecRestoreArray(coordinates, &coords));
388   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
389   PetscCall(VecDestroy(&coordinates));
390   PetscFunctionReturn(PETSC_SUCCESS);
391 }
392 
393 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
394 {
395   const PetscInt numVertices    = (edges[0] + 1) * (edges[1] + 1);
396   const PetscInt numEdges       = edges[0] * (edges[1] + 1) + (edges[0] + 1) * edges[1];
397   PetscInt       markerTop      = 1;
398   PetscInt       markerBottom   = 1;
399   PetscInt       markerRight    = 1;
400   PetscInt       markerLeft     = 1;
401   PetscBool      markerSeparate = PETSC_FALSE;
402   Vec            coordinates;
403   PetscSection   coordSection;
404   PetscScalar   *coords;
405   PetscInt       coordSize;
406   PetscMPIInt    rank;
407   PetscInt       v, vx, vy;
408 
409   PetscFunctionBegin;
410   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
411   if (markerSeparate) {
412     markerTop    = 3;
413     markerBottom = 1;
414     markerRight  = 2;
415     markerLeft   = 4;
416   }
417   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
418   if (rank == 0) {
419     PetscInt e, ex, ey;
420 
421     PetscCall(DMPlexSetChart(dm, 0, numEdges + numVertices));
422     for (e = 0; e < numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
423     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
424     for (vx = 0; vx <= edges[0]; vx++) {
425       for (ey = 0; ey < edges[1]; ey++) {
426         PetscInt edge   = vx * edges[1] + ey + edges[0] * (edges[1] + 1);
427         PetscInt vertex = ey * (edges[0] + 1) + vx + numEdges;
428         PetscInt cone[2];
429 
430         cone[0] = vertex;
431         cone[1] = vertex + edges[0] + 1;
432         PetscCall(DMPlexSetCone(dm, edge, cone));
433         if (vx == edges[0]) {
434           PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
435           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
436           if (ey == edges[1] - 1) {
437             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
438             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerRight));
439           }
440         } else if (vx == 0) {
441           PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
442           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
443           if (ey == edges[1] - 1) {
444             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
445             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft));
446           }
447         }
448       }
449     }
450     for (vy = 0; vy <= edges[1]; vy++) {
451       for (ex = 0; ex < edges[0]; ex++) {
452         PetscInt edge   = vy * edges[0] + ex;
453         PetscInt vertex = vy * (edges[0] + 1) + ex + numEdges;
454         PetscInt cone[2];
455 
456         cone[0] = vertex;
457         cone[1] = vertex + 1;
458         PetscCall(DMPlexSetCone(dm, edge, cone));
459         if (vy == edges[1]) {
460           PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
461           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
462           if (ex == edges[0] - 1) {
463             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
464             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerTop));
465           }
466         } else if (vy == 0) {
467           PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
468           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
469           if (ex == edges[0] - 1) {
470             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
471             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom));
472           }
473         }
474       }
475     }
476   }
477   PetscCall(DMPlexSymmetrize(dm));
478   PetscCall(DMPlexStratify(dm));
479   /* Build coordinates */
480   PetscCall(DMSetCoordinateDim(dm, 2));
481   PetscCall(DMGetCoordinateSection(dm, &coordSection));
482   PetscCall(PetscSectionSetNumFields(coordSection, 1));
483   PetscCall(PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices));
484   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 2));
485   for (v = numEdges; v < numEdges + numVertices; ++v) {
486     PetscCall(PetscSectionSetDof(coordSection, v, 2));
487     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 2));
488   }
489   PetscCall(PetscSectionSetUp(coordSection));
490   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
491   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
492   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
493   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
494   PetscCall(VecSetBlockSize(coordinates, 2));
495   PetscCall(VecSetType(coordinates, VECSTANDARD));
496   PetscCall(VecGetArray(coordinates, &coords));
497   for (vy = 0; vy <= edges[1]; ++vy) {
498     for (vx = 0; vx <= edges[0]; ++vx) {
499       coords[(vy * (edges[0] + 1) + vx) * 2 + 0] = lower[0] + ((upper[0] - lower[0]) / edges[0]) * vx;
500       coords[(vy * (edges[0] + 1) + vx) * 2 + 1] = lower[1] + ((upper[1] - lower[1]) / edges[1]) * vy;
501     }
502   }
503   PetscCall(VecRestoreArray(coordinates, &coords));
504   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
505   PetscCall(VecDestroy(&coordinates));
506   PetscFunctionReturn(PETSC_SUCCESS);
507 }
508 
509 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
510 {
511   PetscInt     vertices[3], numVertices;
512   PetscInt     numFaces       = 2 * faces[0] * faces[1] + 2 * faces[1] * faces[2] + 2 * faces[0] * faces[2];
513   PetscInt     markerTop      = 1;
514   PetscInt     markerBottom   = 1;
515   PetscInt     markerFront    = 1;
516   PetscInt     markerBack     = 1;
517   PetscInt     markerRight    = 1;
518   PetscInt     markerLeft     = 1;
519   PetscBool    markerSeparate = PETSC_FALSE;
520   Vec          coordinates;
521   PetscSection coordSection;
522   PetscScalar *coords;
523   PetscInt     coordSize;
524   PetscMPIInt  rank;
525   PetscInt     v, vx, vy, vz;
526   PetscInt     voffset, iface = 0, cone[4];
527 
528   PetscFunctionBegin;
529   PetscCheck(faces[0] >= 1 && faces[1] >= 1 && faces[2] >= 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
530   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
531   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
532   if (markerSeparate) {
533     markerBottom = 1;
534     markerTop    = 2;
535     markerFront  = 3;
536     markerBack   = 4;
537     markerRight  = 5;
538     markerLeft   = 6;
539   }
540   vertices[0] = faces[0] + 1;
541   vertices[1] = faces[1] + 1;
542   vertices[2] = faces[2] + 1;
543   numVertices = vertices[0] * vertices[1] * vertices[2];
544   if (rank == 0) {
545     PetscInt f;
546 
547     PetscCall(DMPlexSetChart(dm, 0, numFaces + numVertices));
548     for (f = 0; f < numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
549     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
550 
551     /* Side 0 (Top) */
552     for (vy = 0; vy < faces[1]; vy++) {
553       for (vx = 0; vx < faces[0]; vx++) {
554         voffset = numFaces + vertices[0] * vertices[1] * (vertices[2] - 1) + vy * vertices[0] + vx;
555         cone[0] = voffset;
556         cone[1] = voffset + 1;
557         cone[2] = voffset + vertices[0] + 1;
558         cone[3] = voffset + vertices[0];
559         PetscCall(DMPlexSetCone(dm, iface, cone));
560         PetscCall(DMSetLabelValue(dm, "marker", iface, markerTop));
561         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerTop));
562         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerTop));
563         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerTop));
564         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerTop));
565         iface++;
566       }
567     }
568 
569     /* Side 1 (Bottom) */
570     for (vy = 0; vy < faces[1]; vy++) {
571       for (vx = 0; vx < faces[0]; vx++) {
572         voffset = numFaces + vy * (faces[0] + 1) + vx;
573         cone[0] = voffset + 1;
574         cone[1] = voffset;
575         cone[2] = voffset + vertices[0];
576         cone[3] = voffset + vertices[0] + 1;
577         PetscCall(DMPlexSetCone(dm, iface, cone));
578         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBottom));
579         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBottom));
580         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBottom));
581         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerBottom));
582         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerBottom));
583         iface++;
584       }
585     }
586 
587     /* Side 2 (Front) */
588     for (vz = 0; vz < faces[2]; vz++) {
589       for (vx = 0; vx < faces[0]; vx++) {
590         voffset = numFaces + vz * vertices[0] * vertices[1] + vx;
591         cone[0] = voffset;
592         cone[1] = voffset + 1;
593         cone[2] = voffset + vertices[0] * vertices[1] + 1;
594         cone[3] = voffset + vertices[0] * vertices[1];
595         PetscCall(DMPlexSetCone(dm, iface, cone));
596         PetscCall(DMSetLabelValue(dm, "marker", iface, markerFront));
597         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerFront));
598         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerFront));
599         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerFront));
600         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerFront));
601         iface++;
602       }
603     }
604 
605     /* Side 3 (Back) */
606     for (vz = 0; vz < faces[2]; vz++) {
607       for (vx = 0; vx < faces[0]; vx++) {
608         voffset = numFaces + vz * vertices[0] * vertices[1] + vertices[0] * (vertices[1] - 1) + vx;
609         cone[0] = voffset + vertices[0] * vertices[1];
610         cone[1] = voffset + vertices[0] * vertices[1] + 1;
611         cone[2] = voffset + 1;
612         cone[3] = voffset;
613         PetscCall(DMPlexSetCone(dm, iface, cone));
614         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBack));
615         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBack));
616         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBack));
617         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerBack));
618         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerBack));
619         iface++;
620       }
621     }
622 
623     /* Side 4 (Left) */
624     for (vz = 0; vz < faces[2]; vz++) {
625       for (vy = 0; vy < faces[1]; vy++) {
626         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0];
627         cone[0] = voffset;
628         cone[1] = voffset + vertices[0] * vertices[1];
629         cone[2] = voffset + vertices[0] * vertices[1] + vertices[0];
630         cone[3] = voffset + vertices[0];
631         PetscCall(DMPlexSetCone(dm, iface, cone));
632         PetscCall(DMSetLabelValue(dm, "marker", iface, markerLeft));
633         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerLeft));
634         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerLeft));
635         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[1] + 0, markerLeft));
636         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerLeft));
637         iface++;
638       }
639     }
640 
641     /* Side 5 (Right) */
642     for (vz = 0; vz < faces[2]; vz++) {
643       for (vy = 0; vy < faces[1]; vy++) {
644         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0] + faces[0];
645         cone[0] = voffset + vertices[0] * vertices[1];
646         cone[1] = voffset;
647         cone[2] = voffset + vertices[0];
648         cone[3] = voffset + vertices[0] * vertices[1] + vertices[0];
649         PetscCall(DMPlexSetCone(dm, iface, cone));
650         PetscCall(DMSetLabelValue(dm, "marker", iface, markerRight));
651         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerRight));
652         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerRight));
653         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerRight));
654         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerRight));
655         iface++;
656       }
657     }
658   }
659   PetscCall(DMPlexSymmetrize(dm));
660   PetscCall(DMPlexStratify(dm));
661   /* Build coordinates */
662   PetscCall(DMSetCoordinateDim(dm, 3));
663   PetscCall(DMGetCoordinateSection(dm, &coordSection));
664   PetscCall(PetscSectionSetNumFields(coordSection, 1));
665   PetscCall(PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices));
666   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 3));
667   for (v = numFaces; v < numFaces + numVertices; ++v) {
668     PetscCall(PetscSectionSetDof(coordSection, v, 3));
669     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 3));
670   }
671   PetscCall(PetscSectionSetUp(coordSection));
672   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
673   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
674   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
675   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
676   PetscCall(VecSetBlockSize(coordinates, 3));
677   PetscCall(VecSetType(coordinates, VECSTANDARD));
678   PetscCall(VecGetArray(coordinates, &coords));
679   for (vz = 0; vz <= faces[2]; ++vz) {
680     for (vy = 0; vy <= faces[1]; ++vy) {
681       for (vx = 0; vx <= faces[0]; ++vx) {
682         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 0] = lower[0] + ((upper[0] - lower[0]) / faces[0]) * vx;
683         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 1] = lower[1] + ((upper[1] - lower[1]) / faces[1]) * vy;
684         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 2] = lower[2] + ((upper[2] - lower[2]) / faces[2]) * vz;
685       }
686     }
687   }
688   PetscCall(VecRestoreArray(coordinates, &coords));
689   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
690   PetscCall(VecDestroy(&coordinates));
691   PetscFunctionReturn(PETSC_SUCCESS);
692 }
693 
694 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate)
695 {
696   PetscFunctionBegin;
697   PetscValidLogicalCollectiveInt(dm, dim, 2);
698   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
699   PetscCall(DMSetDimension(dm, dim - 1));
700   PetscCall(DMSetCoordinateDim(dm, dim));
701   switch (dim) {
702   case 1:
703     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(dm, lower, upper, faces));
704     break;
705   case 2:
706     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(dm, lower, upper, faces));
707     break;
708   case 3:
709     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(dm, lower, upper, faces));
710     break;
711   default:
712     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Dimension not supported: %" PetscInt_FMT, dim);
713   }
714   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
715   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
716   PetscFunctionReturn(PETSC_SUCCESS);
717 }
718 
719 /*@C
720   DMPlexCreateBoxSurfaceMesh - Creates a mesh on the surface of the tensor product of unit intervals (box) using tensor cells (hexahedra).
721 
722   Collective
723 
724   Input Parameters:
725 + comm        - The communicator for the `DM` object
726 . dim         - The spatial dimension of the box, so the resulting mesh is has dimension `dim`-1
727 . faces       - Number of faces per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
728 . lower       - The lower left corner, or `NULL` for (0, 0, 0)
729 . upper       - The upper right corner, or `NULL` for (1, 1, 1)
730 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
731 
732   Output Parameter:
733 . dm - The `DM` object
734 
735   Level: beginner
736 
737 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateBoxMesh()`, `DMPlexCreateFromFile()`, `DMSetType()`, `DMCreate()`
738 @*/
739 PetscErrorCode DMPlexCreateBoxSurfaceMesh(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate, DM *dm)
740 {
741   PetscInt  fac[3] = {1, 1, 1};
742   PetscReal low[3] = {0, 0, 0};
743   PetscReal upp[3] = {1, 1, 1};
744 
745   PetscFunctionBegin;
746   PetscCall(DMCreate(comm, dm));
747   PetscCall(DMSetType(*dm, DMPLEX));
748   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(*dm, dim, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, interpolate));
749   PetscFunctionReturn(PETSC_SUCCESS);
750 }
751 
752 static PetscErrorCode DMPlexCreateLineMesh_Internal(DM dm, PetscInt segments, PetscReal lower, PetscReal upper, DMBoundaryType bd)
753 {
754   PetscInt     i, fStart, fEnd, numCells = 0, numVerts = 0;
755   PetscInt     numPoints[2], *coneSize, *cones, *coneOrientations;
756   PetscScalar *vertexCoords;
757   PetscReal    L, maxCell;
758   PetscBool    markerSeparate = PETSC_FALSE;
759   PetscInt     markerLeft = 1, faceMarkerLeft = 1;
760   PetscInt     markerRight = 1, faceMarkerRight = 2;
761   PetscBool    wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
762   PetscMPIInt  rank;
763 
764   PetscFunctionBegin;
765   PetscAssertPointer(dm, 1);
766 
767   PetscCall(DMSetDimension(dm, 1));
768   PetscCall(DMCreateLabel(dm, "marker"));
769   PetscCall(DMCreateLabel(dm, "Face Sets"));
770 
771   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
772   if (rank == 0) numCells = segments;
773   if (rank == 0) numVerts = segments + (wrap ? 0 : 1);
774 
775   numPoints[0] = numVerts;
776   numPoints[1] = numCells;
777   PetscCall(PetscMalloc4(numCells + numVerts, &coneSize, numCells * 2, &cones, numCells + numVerts, &coneOrientations, numVerts, &vertexCoords));
778   PetscCall(PetscArrayzero(coneOrientations, numCells + numVerts));
779   for (i = 0; i < numCells; ++i) coneSize[i] = 2;
780   for (i = 0; i < numVerts; ++i) coneSize[numCells + i] = 0;
781   for (i = 0; i < numCells; ++i) {
782     cones[2 * i]     = numCells + i % numVerts;
783     cones[2 * i + 1] = numCells + (i + 1) % numVerts;
784   }
785   for (i = 0; i < numVerts; ++i) vertexCoords[i] = lower + (upper - lower) * ((PetscReal)i / (PetscReal)numCells);
786   PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
787   PetscCall(PetscFree4(coneSize, cones, coneOrientations, vertexCoords));
788 
789   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
790   if (markerSeparate) {
791     markerLeft  = faceMarkerLeft;
792     markerRight = faceMarkerRight;
793   }
794   if (!wrap && rank == 0) {
795     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
796     PetscCall(DMSetLabelValue(dm, "marker", fStart, markerLeft));
797     PetscCall(DMSetLabelValue(dm, "marker", fEnd - 1, markerRight));
798     PetscCall(DMSetLabelValue(dm, "Face Sets", fStart, faceMarkerLeft));
799     PetscCall(DMSetLabelValue(dm, "Face Sets", fEnd - 1, faceMarkerRight));
800   }
801   if (wrap) {
802     L       = upper - lower;
803     maxCell = (PetscReal)1.1 * (L / (PetscReal)PetscMax(1, segments));
804     PetscCall(DMSetPeriodicity(dm, &maxCell, &lower, &L));
805   }
806   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
807   PetscFunctionReturn(PETSC_SUCCESS);
808 }
809 
810 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
811 {
812   DM      boundary, vol;
813   DMLabel bdlabel;
814 
815   PetscFunctionBegin;
816   PetscAssertPointer(dm, 1);
817   for (PetscInt i = 0; i < dim; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
818   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &boundary));
819   PetscCall(DMSetType(boundary, DMPLEX));
820   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(boundary, dim, faces, lower, upper, PETSC_FALSE));
821   PetscCall(DMPlexGenerate(boundary, NULL, interpolate, &vol));
822   PetscCall(DMGetLabel(vol, "marker", &bdlabel));
823   if (bdlabel) PetscCall(DMPlexLabelComplete(vol, bdlabel));
824   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_FALSE, vol));
825   PetscCall(DMPlexReplace_Internal(dm, &vol));
826   PetscCall(DMDestroy(&boundary));
827   PetscFunctionReturn(PETSC_SUCCESS);
828 }
829 
830 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
831 {
832   DMLabel     cutLabel  = NULL;
833   PetscInt    markerTop = 1, faceMarkerTop = 1;
834   PetscInt    markerBottom = 1, faceMarkerBottom = 1;
835   PetscInt    markerFront = 1, faceMarkerFront = 1;
836   PetscInt    markerBack = 1, faceMarkerBack = 1;
837   PetscInt    markerRight = 1, faceMarkerRight = 1;
838   PetscInt    markerLeft = 1, faceMarkerLeft = 1;
839   PetscInt    dim;
840   PetscBool   markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
841   PetscMPIInt rank;
842 
843   PetscFunctionBegin;
844   PetscCall(DMGetDimension(dm, &dim));
845   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
846   PetscCall(DMCreateLabel(dm, "marker"));
847   PetscCall(DMCreateLabel(dm, "Face Sets"));
848   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
849   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST || bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST || bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
850     if (cutMarker) {
851       PetscCall(DMCreateLabel(dm, "periodic_cut"));
852       PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
853     }
854   }
855   switch (dim) {
856   case 2:
857     faceMarkerTop    = 3;
858     faceMarkerBottom = 1;
859     faceMarkerRight  = 2;
860     faceMarkerLeft   = 4;
861     break;
862   case 3:
863     faceMarkerBottom = 1;
864     faceMarkerTop    = 2;
865     faceMarkerFront  = 3;
866     faceMarkerBack   = 4;
867     faceMarkerRight  = 5;
868     faceMarkerLeft   = 6;
869     break;
870   default:
871     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
872   }
873   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
874   if (markerSeparate) {
875     markerBottom = faceMarkerBottom;
876     markerTop    = faceMarkerTop;
877     markerFront  = faceMarkerFront;
878     markerBack   = faceMarkerBack;
879     markerRight  = faceMarkerRight;
880     markerLeft   = faceMarkerLeft;
881   }
882   {
883     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
884     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
885     const PetscInt numZEdges    = rank == 0 ? edges[2] : 0;
886     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0] + 1) : 0;
887     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1] + 1) : 0;
888     const PetscInt numZVertices = rank == 0 ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2] + 1) : 0;
889     const PetscInt numCells     = numXEdges * numYEdges * numZEdges;
890     const PetscInt numXFaces    = numYEdges * numZEdges;
891     const PetscInt numYFaces    = numXEdges * numZEdges;
892     const PetscInt numZFaces    = numXEdges * numYEdges;
893     const PetscInt numTotXFaces = numXVertices * numXFaces;
894     const PetscInt numTotYFaces = numYVertices * numYFaces;
895     const PetscInt numTotZFaces = numZVertices * numZFaces;
896     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
897     const PetscInt numTotXEdges = numXEdges * numYVertices * numZVertices;
898     const PetscInt numTotYEdges = numYEdges * numXVertices * numZVertices;
899     const PetscInt numTotZEdges = numZEdges * numXVertices * numYVertices;
900     const PetscInt numVertices  = numXVertices * numYVertices * numZVertices;
901     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
902     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
903     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
904     const PetscInt firstYFace   = firstXFace + numTotXFaces;
905     const PetscInt firstZFace   = firstYFace + numTotYFaces;
906     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
907     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
908     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
909     Vec            coordinates;
910     PetscSection   coordSection;
911     PetscScalar   *coords;
912     PetscInt       coordSize;
913     PetscInt       v, vx, vy, vz;
914     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
915 
916     PetscCall(DMPlexSetChart(dm, 0, numCells + numFaces + numEdges + numVertices));
917     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
918     for (f = firstXFace; f < firstXFace + numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
919     for (e = firstXEdge; e < firstXEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
920     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
921     /* Build cells */
922     for (fz = 0; fz < numZEdges; ++fz) {
923       for (fy = 0; fy < numYEdges; ++fy) {
924         for (fx = 0; fx < numXEdges; ++fx) {
925           PetscInt cell  = (fz * numYEdges + fy) * numXEdges + fx;
926           PetscInt faceB = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
927           PetscInt faceT = firstZFace + (fy * numXEdges + fx) * numZVertices + ((fz + 1) % numZVertices);
928           PetscInt faceF = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
929           PetscInt faceK = firstYFace + (fz * numXEdges + fx) * numYVertices + ((fy + 1) % numYVertices);
930           PetscInt faceL = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
931           PetscInt faceR = firstXFace + (fz * numYEdges + fy) * numXVertices + ((fx + 1) % numXVertices);
932           /* B,  T,  F,  K,  R,  L */
933           PetscInt ornt[6] = {-2, 0, 0, -3, 0, -2}; /* ??? */
934           PetscInt cone[6];
935 
936           /* no boundary twisting in 3D */
937           cone[0] = faceB;
938           cone[1] = faceT;
939           cone[2] = faceF;
940           cone[3] = faceK;
941           cone[4] = faceR;
942           cone[5] = faceL;
943           PetscCall(DMPlexSetCone(dm, cell, cone));
944           PetscCall(DMPlexSetConeOrientation(dm, cell, ornt));
945           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
946           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
947           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
948         }
949       }
950     }
951     /* Build x faces */
952     for (fz = 0; fz < numZEdges; ++fz) {
953       for (fy = 0; fy < numYEdges; ++fy) {
954         for (fx = 0; fx < numXVertices; ++fx) {
955           PetscInt face    = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
956           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
957           PetscInt edgeR   = firstZEdge + (((fy + 1) % numYVertices) * numXVertices + fx) * numZEdges + fz;
958           PetscInt edgeB   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
959           PetscInt edgeT   = firstYEdge + (((fz + 1) % numZVertices) * numXVertices + fx) * numYEdges + fy;
960           PetscInt ornt[4] = {0, 0, -1, -1};
961           PetscInt cone[4];
962 
963           if (dim == 3) {
964             /* markers */
965             if (bdX != DM_BOUNDARY_PERIODIC) {
966               if (fx == numXVertices - 1) {
967                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight));
968                 PetscCall(DMSetLabelValue(dm, "marker", face, markerRight));
969               } else if (fx == 0) {
970                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft));
971                 PetscCall(DMSetLabelValue(dm, "marker", face, markerLeft));
972               }
973             }
974           }
975           cone[0] = edgeB;
976           cone[1] = edgeR;
977           cone[2] = edgeT;
978           cone[3] = edgeL;
979           PetscCall(DMPlexSetCone(dm, face, cone));
980           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
981         }
982       }
983     }
984     /* Build y faces */
985     for (fz = 0; fz < numZEdges; ++fz) {
986       for (fx = 0; fx < numXEdges; ++fx) {
987         for (fy = 0; fy < numYVertices; ++fy) {
988           PetscInt face    = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
989           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
990           PetscInt edgeR   = firstZEdge + (fy * numXVertices + ((fx + 1) % numXVertices)) * numZEdges + fz;
991           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
992           PetscInt edgeT   = firstXEdge + (((fz + 1) % numZVertices) * numYVertices + fy) * numXEdges + fx;
993           PetscInt ornt[4] = {0, 0, -1, -1};
994           PetscInt cone[4];
995 
996           if (dim == 3) {
997             /* markers */
998             if (bdY != DM_BOUNDARY_PERIODIC) {
999               if (fy == numYVertices - 1) {
1000                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack));
1001                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBack));
1002               } else if (fy == 0) {
1003                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront));
1004                 PetscCall(DMSetLabelValue(dm, "marker", face, markerFront));
1005               }
1006             }
1007           }
1008           cone[0] = edgeB;
1009           cone[1] = edgeR;
1010           cone[2] = edgeT;
1011           cone[3] = edgeL;
1012           PetscCall(DMPlexSetCone(dm, face, cone));
1013           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1014         }
1015       }
1016     }
1017     /* Build z faces */
1018     for (fy = 0; fy < numYEdges; ++fy) {
1019       for (fx = 0; fx < numXEdges; ++fx) {
1020         for (fz = 0; fz < numZVertices; fz++) {
1021           PetscInt face    = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
1022           PetscInt edgeL   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
1023           PetscInt edgeR   = firstYEdge + (fz * numXVertices + ((fx + 1) % numXVertices)) * numYEdges + fy;
1024           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1025           PetscInt edgeT   = firstXEdge + (fz * numYVertices + ((fy + 1) % numYVertices)) * numXEdges + fx;
1026           PetscInt ornt[4] = {0, 0, -1, -1};
1027           PetscInt cone[4];
1028 
1029           if (dim == 2) {
1030             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges - 1) {
1031               edgeR += numYEdges - 1 - 2 * fy;
1032               ornt[1] = -1;
1033             }
1034             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges - 1) {
1035               edgeT += numXEdges - 1 - 2 * fx;
1036               ornt[2] = 0;
1037             }
1038             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1039             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1040           } else {
1041             /* markers */
1042             if (bdZ != DM_BOUNDARY_PERIODIC) {
1043               if (fz == numZVertices - 1) {
1044                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop));
1045                 PetscCall(DMSetLabelValue(dm, "marker", face, markerTop));
1046               } else if (fz == 0) {
1047                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom));
1048                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBottom));
1049               }
1050             }
1051           }
1052           cone[0] = edgeB;
1053           cone[1] = edgeR;
1054           cone[2] = edgeT;
1055           cone[3] = edgeL;
1056           PetscCall(DMPlexSetCone(dm, face, cone));
1057           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1058         }
1059       }
1060     }
1061     /* Build Z edges*/
1062     for (vy = 0; vy < numYVertices; vy++) {
1063       for (vx = 0; vx < numXVertices; vx++) {
1064         for (ez = 0; ez < numZEdges; ez++) {
1065           const PetscInt edge    = firstZEdge + (vy * numXVertices + vx) * numZEdges + ez;
1066           const PetscInt vertexB = firstVertex + (ez * numYVertices + vy) * numXVertices + vx;
1067           const PetscInt vertexT = firstVertex + (((ez + 1) % numZVertices) * numYVertices + vy) * numXVertices + vx;
1068           PetscInt       cone[2];
1069 
1070           cone[0] = vertexB;
1071           cone[1] = vertexT;
1072           PetscCall(DMPlexSetCone(dm, edge, cone));
1073           if (dim == 3) {
1074             if (bdX != DM_BOUNDARY_PERIODIC) {
1075               if (vx == numXVertices - 1) {
1076                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1077                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1078                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1079               } else if (vx == 0) {
1080                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1081                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1082                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1083               }
1084             }
1085             if (bdY != DM_BOUNDARY_PERIODIC) {
1086               if (vy == numYVertices - 1) {
1087                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1088                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1089                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1090               } else if (vy == 0) {
1091                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1092                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1093                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1094               }
1095             }
1096           }
1097         }
1098       }
1099     }
1100     /* Build Y edges*/
1101     for (vz = 0; vz < numZVertices; vz++) {
1102       for (vx = 0; vx < numXVertices; vx++) {
1103         for (ey = 0; ey < numYEdges; ey++) {
1104           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges - 1) ? (numXVertices - vx - 1) : (vz * numYVertices + ((ey + 1) % numYVertices)) * numXVertices + vx;
1105           const PetscInt edge    = firstYEdge + (vz * numXVertices + vx) * numYEdges + ey;
1106           const PetscInt vertexF = firstVertex + (vz * numYVertices + ey) * numXVertices + vx;
1107           const PetscInt vertexK = firstVertex + nextv;
1108           PetscInt       cone[2];
1109 
1110           cone[0] = vertexF;
1111           cone[1] = vertexK;
1112           PetscCall(DMPlexSetCone(dm, edge, cone));
1113           if (dim == 2) {
1114             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1115               if (vx == numXVertices - 1) {
1116                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight));
1117                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1118                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1119                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1120               } else if (vx == 0) {
1121                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft));
1122                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1123                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1124                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1125               }
1126             } else {
1127               if (vx == 0 && cutLabel) {
1128                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1129                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1130                 if (ey == numYEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1131               }
1132             }
1133           } else {
1134             if (bdX != DM_BOUNDARY_PERIODIC) {
1135               if (vx == numXVertices - 1) {
1136                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1137                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1138                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1139               } else if (vx == 0) {
1140                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1141                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1142                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1143               }
1144             }
1145             if (bdZ != DM_BOUNDARY_PERIODIC) {
1146               if (vz == numZVertices - 1) {
1147                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1148                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1149                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1150               } else if (vz == 0) {
1151                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1152                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1153                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1154               }
1155             }
1156           }
1157         }
1158       }
1159     }
1160     /* Build X edges*/
1161     for (vz = 0; vz < numZVertices; vz++) {
1162       for (vy = 0; vy < numYVertices; vy++) {
1163         for (ex = 0; ex < numXEdges; ex++) {
1164           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges - 1) ? (numYVertices - vy - 1) * numXVertices : (vz * numYVertices + vy) * numXVertices + (ex + 1) % numXVertices;
1165           const PetscInt edge    = firstXEdge + (vz * numYVertices + vy) * numXEdges + ex;
1166           const PetscInt vertexL = firstVertex + (vz * numYVertices + vy) * numXVertices + ex;
1167           const PetscInt vertexR = firstVertex + nextv;
1168           PetscInt       cone[2];
1169 
1170           cone[0] = vertexL;
1171           cone[1] = vertexR;
1172           PetscCall(DMPlexSetCone(dm, edge, cone));
1173           if (dim == 2) {
1174             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1175               if (vy == numYVertices - 1) {
1176                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop));
1177                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1178                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1179                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1180               } else if (vy == 0) {
1181                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom));
1182                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1183                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1184                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1185               }
1186             } else {
1187               if (vy == 0 && cutLabel) {
1188                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1189                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1190                 if (ex == numXEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1191               }
1192             }
1193           } else {
1194             if (bdY != DM_BOUNDARY_PERIODIC) {
1195               if (vy == numYVertices - 1) {
1196                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1197                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1198                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1199               } else if (vy == 0) {
1200                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1201                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1202                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1203               }
1204             }
1205             if (bdZ != DM_BOUNDARY_PERIODIC) {
1206               if (vz == numZVertices - 1) {
1207                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1208                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1209                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1210               } else if (vz == 0) {
1211                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1212                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1213                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1214               }
1215             }
1216           }
1217         }
1218       }
1219     }
1220     PetscCall(DMPlexSymmetrize(dm));
1221     PetscCall(DMPlexStratify(dm));
1222     /* Build coordinates */
1223     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1224     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1225     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1226     PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVertices));
1227     for (v = firstVertex; v < firstVertex + numVertices; ++v) {
1228       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1229       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1230     }
1231     PetscCall(PetscSectionSetUp(coordSection));
1232     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1233     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1234     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1235     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1236     PetscCall(VecSetBlockSize(coordinates, dim));
1237     PetscCall(VecSetType(coordinates, VECSTANDARD));
1238     PetscCall(VecGetArray(coordinates, &coords));
1239     for (vz = 0; vz < numZVertices; ++vz) {
1240       for (vy = 0; vy < numYVertices; ++vy) {
1241         for (vx = 0; vx < numXVertices; ++vx) {
1242           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * vx;
1243           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * vy;
1244           if (dim == 3) coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 2] = lower[2] + ((upper[2] - lower[2]) / numZEdges) * vz;
1245         }
1246       }
1247     }
1248     PetscCall(VecRestoreArray(coordinates, &coords));
1249     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1250     PetscCall(VecDestroy(&coordinates));
1251   }
1252   PetscFunctionReturn(PETSC_SUCCESS);
1253 }
1254 
1255 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1256 {
1257   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1258   PetscInt       fac[3] = {0, 0, 0}, d;
1259 
1260   PetscFunctionBegin;
1261   PetscAssertPointer(dm, 1);
1262   PetscValidLogicalCollectiveInt(dm, dim, 2);
1263   PetscCall(DMSetDimension(dm, dim));
1264   for (d = 0; d < dim; ++d) {
1265     fac[d] = faces[d];
1266     bdt[d] = periodicity[d];
1267   }
1268   PetscCall(DMPlexCreateCubeMesh_Internal(dm, lower, upper, fac, bdt[0], bdt[1], bdt[2]));
1269   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST || periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST || (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
1270     PetscReal L[3]       = {-1., -1., 0.};
1271     PetscReal maxCell[3] = {-1., -1., 0.};
1272 
1273     for (d = 0; d < dim; ++d) {
1274       if (periodicity[d] != DM_BOUNDARY_NONE) {
1275         L[d]       = upper[d] - lower[d];
1276         maxCell[d] = 1.1 * (L[d] / PetscMax(1, faces[d]));
1277       }
1278     }
1279     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1280   }
1281   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
1282   PetscFunctionReturn(PETSC_SUCCESS);
1283 }
1284 
1285 static PetscErrorCode DMPlexCreateBoxMesh_Internal(DM dm, DMPlexShape shape, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1286 {
1287   PetscFunctionBegin;
1288   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
1289   if (shape == DM_SHAPE_ZBOX) PetscCall(DMPlexCreateBoxMesh_Tensor_SFC_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1290   else if (dim == 1) PetscCall(DMPlexCreateLineMesh_Internal(dm, faces[0], lower[0], upper[0], periodicity[0]));
1291   else if (simplex) PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1292   else PetscCall(DMPlexCreateBoxMesh_Tensor_Internal(dm, dim, faces, lower, upper, periodicity));
1293   if (!interpolate && dim > 1 && !simplex) {
1294     DM udm;
1295 
1296     PetscCall(DMPlexUninterpolate(dm, &udm));
1297     PetscCall(DMPlexCopyCoordinates(dm, udm));
1298     PetscCall(DMPlexReplace_Internal(dm, &udm));
1299   }
1300   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
1301   PetscFunctionReturn(PETSC_SUCCESS);
1302 }
1303 
1304 /*@C
1305   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
1306 
1307   Collective
1308 
1309   Input Parameters:
1310 + comm        - The communicator for the `DM` object
1311 . dim         - The spatial dimension
1312 . simplex     - `PETSC_TRUE` for simplices, `PETSC_FALSE` for tensor cells
1313 . faces       - Number of faces per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1314 . lower       - The lower left corner, or `NULL` for (0, 0, 0)
1315 . upper       - The upper right corner, or `NULL` for (1, 1, 1)
1316 . periodicity - The boundary type for the X,Y,Z direction, or `NULL` for `DM_BOUNDARY_NONE`
1317 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1318 
1319   Output Parameter:
1320 . dm - The `DM` object
1321 
1322   Level: beginner
1323 
1324   Note:
1325   To customize this mesh using options, use
1326 .vb
1327   DMCreate(comm, &dm);
1328   DMSetType(dm, DMPLEX);
1329   DMSetFromOptions(dm);
1330 .ve
1331   and use the options in `DMSetFromOptions()`.
1332 
1333   Here is the numbering returned for 2 faces in each direction for tensor cells\:
1334 .vb
1335  10---17---11---18----12
1336   |         |         |
1337   |         |         |
1338  20    2   22    3    24
1339   |         |         |
1340   |         |         |
1341   7---15----8---16----9
1342   |         |         |
1343   |         |         |
1344  19    0   21    1   23
1345   |         |         |
1346   |         |         |
1347   4---13----5---14----6
1348 .ve
1349   and for simplicial cells
1350 .vb
1351  14----8---15----9----16
1352   |\     5  |\      7 |
1353   | \       | \       |
1354  13   2    14    3    15
1355   | 4   \   | 6   \   |
1356   |       \ |       \ |
1357  11----6---12----7----13
1358   |\        |\        |
1359   | \    1  | \     3 |
1360  10   0    11    1    12
1361   | 0   \   | 2   \   |
1362   |       \ |       \ |
1363   8----4----9----5----10
1364 .ve
1365 
1366 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
1367 @*/
1368 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
1369 {
1370   PetscInt       fac[3] = {1, 1, 1};
1371   PetscReal      low[3] = {0, 0, 0};
1372   PetscReal      upp[3] = {1, 1, 1};
1373   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1374 
1375   PetscFunctionBegin;
1376   PetscCall(DMCreate(comm, dm));
1377   PetscCall(DMSetType(*dm, DMPLEX));
1378   PetscCall(DMPlexCreateBoxMesh_Internal(*dm, DM_SHAPE_BOX, dim, simplex, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt, interpolate));
1379   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1380   PetscFunctionReturn(PETSC_SUCCESS);
1381 }
1382 
1383 static PetscErrorCode DMPlexCreateWedgeBoxMesh_Internal(DM dm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1384 {
1385   DM       bdm, vol;
1386   PetscInt i;
1387 
1388   PetscFunctionBegin;
1389   // TODO Now we can support periodicity
1390   for (i = 0; i < 3; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity not yet supported");
1391   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &bdm));
1392   PetscCall(DMSetType(bdm, DMPLEX));
1393   PetscCall(DMSetDimension(bdm, 2));
1394   PetscCall(PetscLogEventBegin(DMPLEX_Generate, bdm, 0, 0, 0));
1395   PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(bdm, 2, faces, lower, upper, periodicity, PETSC_TRUE));
1396   PetscCall(DMPlexExtrude(bdm, faces[2], upper[2] - lower[2], PETSC_TRUE, PETSC_FALSE, PETSC_FALSE, NULL, NULL, &vol));
1397   PetscCall(PetscLogEventEnd(DMPLEX_Generate, bdm, 0, 0, 0));
1398   PetscCall(DMDestroy(&bdm));
1399   PetscCall(DMPlexReplace_Internal(dm, &vol));
1400   if (lower[2] != 0.0) {
1401     Vec          v;
1402     PetscScalar *x;
1403     PetscInt     cDim, n;
1404 
1405     PetscCall(DMGetCoordinatesLocal(dm, &v));
1406     PetscCall(VecGetBlockSize(v, &cDim));
1407     PetscCall(VecGetLocalSize(v, &n));
1408     PetscCall(VecGetArray(v, &x));
1409     x += cDim;
1410     for (i = 0; i < n; i += cDim) x[i] += lower[2];
1411     PetscCall(VecRestoreArray(v, &x));
1412     PetscCall(DMSetCoordinatesLocal(dm, v));
1413   }
1414   PetscFunctionReturn(PETSC_SUCCESS);
1415 }
1416 
1417 /*@
1418   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tesselating the (x,y) plane and extruding in the third direction using wedge cells.
1419 
1420   Collective
1421 
1422   Input Parameters:
1423 + comm        - The communicator for the `DM` object
1424 . faces       - Number of faces per dimension, or `NULL` for (1, 1, 1)
1425 . lower       - The lower left corner, or `NULL` for (0, 0, 0)
1426 . upper       - The upper right corner, or `NULL` for (1, 1, 1)
1427 . periodicity - The boundary type for the X,Y,Z direction, or `NULL` for `DM_BOUNDARY_NONE`
1428 . orderHeight - If `PETSC_TRUE`, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1429 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1430 
1431   Output Parameter:
1432 . dm - The `DM` object
1433 
1434   Level: beginner
1435 
1436 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateWedgeCylinderMesh()`, `DMExtrude()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1437 @*/
1438 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool orderHeight, PetscBool interpolate, DM *dm)
1439 {
1440   PetscInt       fac[3] = {1, 1, 1};
1441   PetscReal      low[3] = {0, 0, 0};
1442   PetscReal      upp[3] = {1, 1, 1};
1443   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1444 
1445   PetscFunctionBegin;
1446   PetscCall(DMCreate(comm, dm));
1447   PetscCall(DMSetType(*dm, DMPLEX));
1448   PetscCall(DMPlexCreateWedgeBoxMesh_Internal(*dm, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt));
1449   if (!interpolate) {
1450     DM udm;
1451 
1452     PetscCall(DMPlexUninterpolate(*dm, &udm));
1453     PetscCall(DMPlexReplace_Internal(*dm, &udm));
1454   }
1455   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1456   PetscFunctionReturn(PETSC_SUCCESS);
1457 }
1458 
1459 /*
1460   DMPlexTensorPointLexicographic_Private - Returns all tuples of size 'len' with nonnegative integers that are all less than or equal to 'max' for that dimension.
1461 
1462   Input Parameters:
1463 + len - The length of the tuple
1464 . max - The maximum for each dimension, so values are in [0, max)
1465 - tup - A tuple of length len+1: tup[len] > 0 indicates a stopping condition
1466 
1467   Output Parameter:
1468 . tup - A tuple of `len` integers whose entries are at most `max`
1469 
1470   Level: developer
1471 
1472   Note:
1473   Ordering is lexicographic with lowest index as least significant in ordering.
1474   e.g. for len == 2 and max == 2, this will return, in order, {0,0}, {1,0}, {2,0}, {0,1}, {1,1}, {2,1}, {0,2}, {1,2}, {2,2}.
1475 
1476 .seealso: PetscDualSpaceTensorPointLexicographic_Internal(), PetscDualSpaceLatticePointLexicographic_Internal()
1477 */
1478 static PetscErrorCode DMPlexTensorPointLexicographic_Private(PetscInt len, const PetscInt max[], PetscInt tup[])
1479 {
1480   PetscInt i;
1481 
1482   PetscFunctionBegin;
1483   for (i = 0; i < len; ++i) {
1484     if (tup[i] < max[i] - 1) {
1485       break;
1486     } else {
1487       tup[i] = 0;
1488     }
1489   }
1490   if (i == len) tup[i - 1] = max[i - 1];
1491   else ++tup[i];
1492   PetscFunctionReturn(PETSC_SUCCESS);
1493 }
1494 
1495 static PetscInt TupleToIndex_Private(PetscInt len, const PetscInt max[], const PetscInt tup[])
1496 {
1497   PetscInt i, idx = tup[len - 1];
1498 
1499   for (i = len - 2; i >= 0; --i) {
1500     idx *= max[i];
1501     idx += tup[i];
1502   }
1503   return idx;
1504 }
1505 
1506 static PetscErrorCode DestroyExtent_Private(void *extent)
1507 {
1508   return PetscFree(extent);
1509 }
1510 
1511 static PetscErrorCode DMPlexCreateHypercubicMesh_Internal(DM dm, PetscInt dim, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], const DMBoundaryType bd[])
1512 {
1513   Vec          coordinates;
1514   PetscSection coordSection;
1515   DMLabel      cutLabel    = NULL;
1516   PetscBool    cutMarker   = PETSC_FALSE;
1517   PetscBool    periodic    = PETSC_FALSE;
1518   PetscInt     numCells    = 1, c;
1519   PetscInt     numVertices = 1, v;
1520   PetscScalar *coords;
1521   PetscInt    *vertices, *vert, *vtmp, *supp, cone[2];
1522   PetscInt     d, e, cell = 0, coordSize;
1523   PetscMPIInt  rank;
1524 
1525   PetscFunctionBegin;
1526   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1527   PetscCall(DMSetDimension(dm, dim));
1528   PetscCall(PetscCalloc4(dim, &vertices, dim, &vert, dim, &vtmp, 2 * dim, &supp));
1529   PetscCall(DMCreateLabel(dm, "marker"));
1530   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
1531   for (d = 0; d < dim; ++d) periodic = (periodic || bd[d] == DM_BOUNDARY_PERIODIC) ? PETSC_TRUE : PETSC_FALSE;
1532   if (periodic && cutMarker) {
1533     PetscCall(DMCreateLabel(dm, "periodic_cut"));
1534     PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
1535   }
1536   for (d = 0; d < dim; ++d) PetscCheck(bd[d] == DM_BOUNDARY_PERIODIC, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Hypercubic mesh must be periodic now");
1537   for (d = 0; d < dim; ++d) {
1538     vertices[d] = edges[d];
1539     numVertices *= vertices[d];
1540   }
1541   numCells = numVertices * dim;
1542   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1543   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, 2));
1544   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetSupportSize(dm, v, 2 * dim));
1545   /* TODO Loop over boundary and reset support sizes */
1546   PetscCall(DMSetUp(dm)); /* Allocate space for cones and supports */
1547   /* Build cell cones and vertex supports */
1548   PetscCall(DMCreateLabel(dm, "celltype"));
1549   while (vert[dim - 1] < vertices[dim - 1]) {
1550     const PetscInt vertex = TupleToIndex_Private(dim, vertices, vert) + numCells;
1551     PetscInt       s      = 0;
1552 
1553     PetscCall(PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT ":", vertex));
1554     for (d = 0; d < dim; ++d) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, vert[d]));
1555     PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
1556     PetscCall(DMPlexSetCellType(dm, vertex, DM_POLYTOPE_POINT));
1557     for (d = 0; d < dim; ++d) {
1558       for (e = 0; e < dim; ++e) vtmp[e] = vert[e];
1559       vtmp[d] = (vert[d] + 1) % vertices[d];
1560       cone[0] = vertex;
1561       cone[1] = TupleToIndex_Private(dim, vertices, vtmp) + numCells;
1562       PetscCall(PetscPrintf(PETSC_COMM_SELF, "  Vertex %" PetscInt_FMT ":", cone[1]));
1563       for (e = 0; e < dim; ++e) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, vtmp[e]));
1564       PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
1565       PetscCall(DMPlexSetCone(dm, cell, cone));
1566       PetscCall(DMPlexSetCellType(dm, cell, DM_POLYTOPE_SEGMENT));
1567       PetscCall(PetscPrintf(PETSC_COMM_SELF, "  Edge %" PetscInt_FMT " (%" PetscInt_FMT " %" PetscInt_FMT ")\n", cell, cone[0], cone[1]));
1568       ++cell;
1569     }
1570     for (d = 0; d < dim; ++d) {
1571       for (e = 0; e < dim; ++e) vtmp[e] = vert[e];
1572       vtmp[d]   = (vert[d] + vertices[d] - 1) % vertices[d];
1573       supp[s++] = TupleToIndex_Private(dim, vertices, vtmp) * dim + d;
1574       supp[s++] = (vertex - numCells) * dim + d;
1575       PetscCall(DMPlexSetSupport(dm, vertex, supp));
1576     }
1577     PetscCall(DMPlexTensorPointLexicographic_Private(dim, vertices, vert));
1578   }
1579   PetscCall(DMPlexStratify(dm));
1580   /* Build coordinates */
1581   PetscCall(DMGetCoordinateSection(dm, &coordSection));
1582   PetscCall(PetscSectionSetNumFields(coordSection, 1));
1583   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1584   PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1585   for (v = numCells; v < numCells + numVertices; ++v) {
1586     PetscCall(PetscSectionSetDof(coordSection, v, dim));
1587     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1588   }
1589   PetscCall(PetscSectionSetUp(coordSection));
1590   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1591   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1592   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1593   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1594   PetscCall(VecSetBlockSize(coordinates, dim));
1595   PetscCall(VecSetType(coordinates, VECSTANDARD));
1596   PetscCall(VecGetArray(coordinates, &coords));
1597   for (d = 0; d < dim; ++d) vert[d] = 0;
1598   while (vert[dim - 1] < vertices[dim - 1]) {
1599     const PetscInt vertex = TupleToIndex_Private(dim, vertices, vert);
1600 
1601     for (d = 0; d < dim; ++d) coords[vertex * dim + d] = lower[d] + ((upper[d] - lower[d]) / vertices[d]) * vert[d];
1602     PetscCall(PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT ":", vertex));
1603     for (d = 0; d < dim; ++d) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, vert[d]));
1604     for (d = 0; d < dim; ++d) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %g", (double)PetscRealPart(coords[vertex * dim + d])));
1605     PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
1606     PetscCall(DMPlexTensorPointLexicographic_Private(dim, vertices, vert));
1607   }
1608   PetscCall(VecRestoreArray(coordinates, &coords));
1609   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1610   PetscCall(VecDestroy(&coordinates));
1611   PetscCall(PetscFree4(vertices, vert, vtmp, supp));
1612   //PetscCall(DMSetPeriodicity(dm, NULL, lower, upper));
1613   // Attach the extent
1614   {
1615     PetscContainer c;
1616     PetscInt      *extent;
1617 
1618     PetscCall(PetscMalloc1(dim, &extent));
1619     for (PetscInt d = 0; d < dim; ++d) extent[d] = edges[d];
1620     PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &c));
1621     PetscCall(PetscContainerSetUserDestroy(c, DestroyExtent_Private));
1622     PetscCall(PetscContainerSetPointer(c, extent));
1623     PetscCall(PetscObjectCompose((PetscObject)dm, "_extent", (PetscObject)c));
1624     PetscCall(PetscContainerDestroy(&c));
1625   }
1626   PetscFunctionReturn(PETSC_SUCCESS);
1627 }
1628 
1629 /*@C
1630   DMPlexCreateHypercubicMesh - Creates a periodic mesh on the tensor product of unit intervals using only vertices and edges.
1631 
1632   Collective
1633 
1634   Input Parameters:
1635 + comm  - The communicator for the DM object
1636 . dim   - The spatial dimension
1637 . edges - Number of edges per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1638 . lower - The lower left corner, or `NULL` for (0, 0, 0)
1639 - upper - The upper right corner, or `NULL` for (1, 1, 1)
1640 
1641   Output Parameter:
1642 . dm - The DM object
1643 
1644   Level: beginner
1645 
1646   Note:
1647   If you want to customize this mesh using options, you just need to
1648 .vb
1649   DMCreate(comm, &dm);
1650   DMSetType(dm, DMPLEX);
1651   DMSetFromOptions(dm);
1652 .ve
1653   and use the options on the `DMSetFromOptions()` page.
1654 
1655   The vertices are numbered is lexicographic order, and the dim edges exiting a vertex in the positive orthant are number consecutively,
1656 .vb
1657  18--0-19--2-20--4-18
1658   |     |     |     |
1659  13    15    17    13
1660   |     |     |     |
1661  24-12-25-14-26-16-24
1662   |     |     |     |
1663   7     9    11     7
1664   |     |     |     |
1665  21--6-22--8-23-10-21
1666   |     |     |     |
1667   1     3     5     1
1668   |     |     |     |
1669  18--0-19--2-20--4-18
1670 .ve
1671 
1672 .seealso: `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
1673 @*/
1674 PetscErrorCode DMPlexCreateHypercubicMesh(MPI_Comm comm, PetscInt dim, const PetscInt edges[], const PetscReal lower[], const PetscReal upper[], DM *dm)
1675 {
1676   PetscInt       *edg;
1677   PetscReal      *low, *upp;
1678   DMBoundaryType *bdt;
1679   PetscInt        d;
1680 
1681   PetscFunctionBegin;
1682   PetscCall(DMCreate(comm, dm));
1683   PetscCall(DMSetType(*dm, DMPLEX));
1684   PetscCall(PetscMalloc4(dim, &edg, dim, &low, dim, &upp, dim, &bdt));
1685   for (d = 0; d < dim; ++d) {
1686     edg[d] = edges ? edges[d] : 1;
1687     low[d] = lower ? lower[d] : 0.;
1688     upp[d] = upper ? upper[d] : 1.;
1689     bdt[d] = DM_BOUNDARY_PERIODIC;
1690   }
1691   PetscCall(DMPlexCreateHypercubicMesh_Internal(*dm, dim, low, upp, edg, bdt));
1692   PetscCall(PetscFree4(edg, low, upp, bdt));
1693   PetscFunctionReturn(PETSC_SUCCESS);
1694 }
1695 
1696 /*@C
1697   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all `DM` options in the database.
1698 
1699   Logically Collective
1700 
1701   Input Parameters:
1702 + dm     - the `DM` context
1703 - prefix - the prefix to prepend to all option names
1704 
1705   Level: advanced
1706 
1707   Note:
1708   A hyphen (-) must NOT be given at the beginning of the prefix name.
1709   The first character of all runtime options is AUTOMATICALLY the hyphen.
1710 
1711 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `SNESSetFromOptions()`
1712 @*/
1713 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
1714 {
1715   DM_Plex *mesh = (DM_Plex *)dm->data;
1716 
1717   PetscFunctionBegin;
1718   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1719   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, prefix));
1720   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)mesh->partitioner, prefix));
1721   PetscFunctionReturn(PETSC_SUCCESS);
1722 }
1723 
1724 /* Remap geometry to cylinder
1725    TODO: This only works for a single refinement, then it is broken
1726 
1727      Interior square: Linear interpolation is correct
1728      The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
1729      such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
1730 
1731        phi     = arctan(y/x)
1732        d_close = sqrt(1/8 + 1/4 sin^2(phi))
1733        d_far   = sqrt(1/2 + sin^2(phi))
1734 
1735      so we remap them using
1736 
1737        x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
1738        y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
1739 
1740      If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
1741 */
1742 static void snapToCylinder(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
1743 {
1744   const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1745   const PetscReal ds2 = 0.5 * dis;
1746 
1747   if ((PetscAbsScalar(u[0]) <= ds2) && (PetscAbsScalar(u[1]) <= ds2)) {
1748     f0[0] = u[0];
1749     f0[1] = u[1];
1750   } else {
1751     PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
1752 
1753     x    = PetscRealPart(u[0]);
1754     y    = PetscRealPart(u[1]);
1755     phi  = PetscAtan2Real(y, x);
1756     sinp = PetscSinReal(phi);
1757     cosp = PetscCosReal(phi);
1758     if ((PetscAbsReal(phi) > PETSC_PI / 4.0) && (PetscAbsReal(phi) < 3.0 * PETSC_PI / 4.0)) {
1759       dc = PetscAbsReal(ds2 / sinp);
1760       df = PetscAbsReal(dis / sinp);
1761       xc = ds2 * x / PetscAbsReal(y);
1762       yc = ds2 * PetscSignReal(y);
1763     } else {
1764       dc = PetscAbsReal(ds2 / cosp);
1765       df = PetscAbsReal(dis / cosp);
1766       xc = ds2 * PetscSignReal(x);
1767       yc = ds2 * y / PetscAbsReal(x);
1768     }
1769     f0[0] = xc + (u[0] - xc) * (1.0 - dc) / (df - dc);
1770     f0[1] = yc + (u[1] - yc) * (1.0 - dc) / (df - dc);
1771   }
1772   f0[2] = u[2];
1773 }
1774 
1775 static PetscErrorCode DMPlexCreateHexCylinderMesh_Internal(DM dm, DMBoundaryType periodicZ)
1776 {
1777   const PetscInt dim = 3;
1778   PetscInt       numCells, numVertices;
1779   PetscMPIInt    rank;
1780 
1781   PetscFunctionBegin;
1782   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
1783   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1784   PetscCall(DMSetDimension(dm, dim));
1785   /* Create topology */
1786   {
1787     PetscInt cone[8], c;
1788 
1789     numCells    = rank == 0 ? 5 : 0;
1790     numVertices = rank == 0 ? 16 : 0;
1791     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1792       numCells *= 3;
1793       numVertices = rank == 0 ? 24 : 0;
1794     }
1795     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1796     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 8));
1797     PetscCall(DMSetUp(dm));
1798     if (rank == 0) {
1799       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1800         cone[0] = 15;
1801         cone[1] = 18;
1802         cone[2] = 17;
1803         cone[3] = 16;
1804         cone[4] = 31;
1805         cone[5] = 32;
1806         cone[6] = 33;
1807         cone[7] = 34;
1808         PetscCall(DMPlexSetCone(dm, 0, cone));
1809         cone[0] = 16;
1810         cone[1] = 17;
1811         cone[2] = 24;
1812         cone[3] = 23;
1813         cone[4] = 32;
1814         cone[5] = 36;
1815         cone[6] = 37;
1816         cone[7] = 33; /* 22 25 26 21 */
1817         PetscCall(DMPlexSetCone(dm, 1, cone));
1818         cone[0] = 18;
1819         cone[1] = 27;
1820         cone[2] = 24;
1821         cone[3] = 17;
1822         cone[4] = 34;
1823         cone[5] = 33;
1824         cone[6] = 37;
1825         cone[7] = 38;
1826         PetscCall(DMPlexSetCone(dm, 2, cone));
1827         cone[0] = 29;
1828         cone[1] = 27;
1829         cone[2] = 18;
1830         cone[3] = 15;
1831         cone[4] = 35;
1832         cone[5] = 31;
1833         cone[6] = 34;
1834         cone[7] = 38;
1835         PetscCall(DMPlexSetCone(dm, 3, cone));
1836         cone[0] = 29;
1837         cone[1] = 15;
1838         cone[2] = 16;
1839         cone[3] = 23;
1840         cone[4] = 35;
1841         cone[5] = 36;
1842         cone[6] = 32;
1843         cone[7] = 31;
1844         PetscCall(DMPlexSetCone(dm, 4, cone));
1845 
1846         cone[0] = 31;
1847         cone[1] = 34;
1848         cone[2] = 33;
1849         cone[3] = 32;
1850         cone[4] = 19;
1851         cone[5] = 22;
1852         cone[6] = 21;
1853         cone[7] = 20;
1854         PetscCall(DMPlexSetCone(dm, 5, cone));
1855         cone[0] = 32;
1856         cone[1] = 33;
1857         cone[2] = 37;
1858         cone[3] = 36;
1859         cone[4] = 22;
1860         cone[5] = 25;
1861         cone[6] = 26;
1862         cone[7] = 21;
1863         PetscCall(DMPlexSetCone(dm, 6, cone));
1864         cone[0] = 34;
1865         cone[1] = 38;
1866         cone[2] = 37;
1867         cone[3] = 33;
1868         cone[4] = 20;
1869         cone[5] = 21;
1870         cone[6] = 26;
1871         cone[7] = 28;
1872         PetscCall(DMPlexSetCone(dm, 7, cone));
1873         cone[0] = 35;
1874         cone[1] = 38;
1875         cone[2] = 34;
1876         cone[3] = 31;
1877         cone[4] = 30;
1878         cone[5] = 19;
1879         cone[6] = 20;
1880         cone[7] = 28;
1881         PetscCall(DMPlexSetCone(dm, 8, cone));
1882         cone[0] = 35;
1883         cone[1] = 31;
1884         cone[2] = 32;
1885         cone[3] = 36;
1886         cone[4] = 30;
1887         cone[5] = 25;
1888         cone[6] = 22;
1889         cone[7] = 19;
1890         PetscCall(DMPlexSetCone(dm, 9, cone));
1891 
1892         cone[0] = 19;
1893         cone[1] = 20;
1894         cone[2] = 21;
1895         cone[3] = 22;
1896         cone[4] = 15;
1897         cone[5] = 16;
1898         cone[6] = 17;
1899         cone[7] = 18;
1900         PetscCall(DMPlexSetCone(dm, 10, cone));
1901         cone[0] = 22;
1902         cone[1] = 21;
1903         cone[2] = 26;
1904         cone[3] = 25;
1905         cone[4] = 16;
1906         cone[5] = 23;
1907         cone[6] = 24;
1908         cone[7] = 17;
1909         PetscCall(DMPlexSetCone(dm, 11, cone));
1910         cone[0] = 20;
1911         cone[1] = 28;
1912         cone[2] = 26;
1913         cone[3] = 21;
1914         cone[4] = 18;
1915         cone[5] = 17;
1916         cone[6] = 24;
1917         cone[7] = 27;
1918         PetscCall(DMPlexSetCone(dm, 12, cone));
1919         cone[0] = 30;
1920         cone[1] = 28;
1921         cone[2] = 20;
1922         cone[3] = 19;
1923         cone[4] = 29;
1924         cone[5] = 15;
1925         cone[6] = 18;
1926         cone[7] = 27;
1927         PetscCall(DMPlexSetCone(dm, 13, cone));
1928         cone[0] = 30;
1929         cone[1] = 19;
1930         cone[2] = 22;
1931         cone[3] = 25;
1932         cone[4] = 29;
1933         cone[5] = 23;
1934         cone[6] = 16;
1935         cone[7] = 15;
1936         PetscCall(DMPlexSetCone(dm, 14, cone));
1937       } else {
1938         cone[0] = 5;
1939         cone[1] = 8;
1940         cone[2] = 7;
1941         cone[3] = 6;
1942         cone[4] = 9;
1943         cone[5] = 12;
1944         cone[6] = 11;
1945         cone[7] = 10;
1946         PetscCall(DMPlexSetCone(dm, 0, cone));
1947         cone[0] = 6;
1948         cone[1] = 7;
1949         cone[2] = 14;
1950         cone[3] = 13;
1951         cone[4] = 12;
1952         cone[5] = 15;
1953         cone[6] = 16;
1954         cone[7] = 11;
1955         PetscCall(DMPlexSetCone(dm, 1, cone));
1956         cone[0] = 8;
1957         cone[1] = 17;
1958         cone[2] = 14;
1959         cone[3] = 7;
1960         cone[4] = 10;
1961         cone[5] = 11;
1962         cone[6] = 16;
1963         cone[7] = 18;
1964         PetscCall(DMPlexSetCone(dm, 2, cone));
1965         cone[0] = 19;
1966         cone[1] = 17;
1967         cone[2] = 8;
1968         cone[3] = 5;
1969         cone[4] = 20;
1970         cone[5] = 9;
1971         cone[6] = 10;
1972         cone[7] = 18;
1973         PetscCall(DMPlexSetCone(dm, 3, cone));
1974         cone[0] = 19;
1975         cone[1] = 5;
1976         cone[2] = 6;
1977         cone[3] = 13;
1978         cone[4] = 20;
1979         cone[5] = 15;
1980         cone[6] = 12;
1981         cone[7] = 9;
1982         PetscCall(DMPlexSetCone(dm, 4, cone));
1983       }
1984     }
1985     PetscCall(DMPlexSymmetrize(dm));
1986     PetscCall(DMPlexStratify(dm));
1987   }
1988   /* Create cube geometry */
1989   {
1990     Vec             coordinates;
1991     PetscSection    coordSection;
1992     PetscScalar    *coords;
1993     PetscInt        coordSize, v;
1994     const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1995     const PetscReal ds2 = dis / 2.0;
1996 
1997     /* Build coordinates */
1998     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1999     PetscCall(PetscSectionSetNumFields(coordSection, 1));
2000     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
2001     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
2002     for (v = numCells; v < numCells + numVertices; ++v) {
2003       PetscCall(PetscSectionSetDof(coordSection, v, dim));
2004       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
2005     }
2006     PetscCall(PetscSectionSetUp(coordSection));
2007     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2008     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2009     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2010     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2011     PetscCall(VecSetBlockSize(coordinates, dim));
2012     PetscCall(VecSetType(coordinates, VECSTANDARD));
2013     PetscCall(VecGetArray(coordinates, &coords));
2014     if (rank == 0) {
2015       coords[0 * dim + 0]  = -ds2;
2016       coords[0 * dim + 1]  = -ds2;
2017       coords[0 * dim + 2]  = 0.0;
2018       coords[1 * dim + 0]  = ds2;
2019       coords[1 * dim + 1]  = -ds2;
2020       coords[1 * dim + 2]  = 0.0;
2021       coords[2 * dim + 0]  = ds2;
2022       coords[2 * dim + 1]  = ds2;
2023       coords[2 * dim + 2]  = 0.0;
2024       coords[3 * dim + 0]  = -ds2;
2025       coords[3 * dim + 1]  = ds2;
2026       coords[3 * dim + 2]  = 0.0;
2027       coords[4 * dim + 0]  = -ds2;
2028       coords[4 * dim + 1]  = -ds2;
2029       coords[4 * dim + 2]  = 1.0;
2030       coords[5 * dim + 0]  = -ds2;
2031       coords[5 * dim + 1]  = ds2;
2032       coords[5 * dim + 2]  = 1.0;
2033       coords[6 * dim + 0]  = ds2;
2034       coords[6 * dim + 1]  = ds2;
2035       coords[6 * dim + 2]  = 1.0;
2036       coords[7 * dim + 0]  = ds2;
2037       coords[7 * dim + 1]  = -ds2;
2038       coords[7 * dim + 2]  = 1.0;
2039       coords[8 * dim + 0]  = dis;
2040       coords[8 * dim + 1]  = -dis;
2041       coords[8 * dim + 2]  = 0.0;
2042       coords[9 * dim + 0]  = dis;
2043       coords[9 * dim + 1]  = dis;
2044       coords[9 * dim + 2]  = 0.0;
2045       coords[10 * dim + 0] = dis;
2046       coords[10 * dim + 1] = -dis;
2047       coords[10 * dim + 2] = 1.0;
2048       coords[11 * dim + 0] = dis;
2049       coords[11 * dim + 1] = dis;
2050       coords[11 * dim + 2] = 1.0;
2051       coords[12 * dim + 0] = -dis;
2052       coords[12 * dim + 1] = dis;
2053       coords[12 * dim + 2] = 0.0;
2054       coords[13 * dim + 0] = -dis;
2055       coords[13 * dim + 1] = dis;
2056       coords[13 * dim + 2] = 1.0;
2057       coords[14 * dim + 0] = -dis;
2058       coords[14 * dim + 1] = -dis;
2059       coords[14 * dim + 2] = 0.0;
2060       coords[15 * dim + 0] = -dis;
2061       coords[15 * dim + 1] = -dis;
2062       coords[15 * dim + 2] = 1.0;
2063       if (periodicZ == DM_BOUNDARY_PERIODIC) {
2064         /* 15 31 19 */ coords[16 * dim + 0] = -ds2;
2065         coords[16 * dim + 1]                = -ds2;
2066         coords[16 * dim + 2]                = 0.5;
2067         /* 16 32 22 */ coords[17 * dim + 0] = ds2;
2068         coords[17 * dim + 1]                = -ds2;
2069         coords[17 * dim + 2]                = 0.5;
2070         /* 17 33 21 */ coords[18 * dim + 0] = ds2;
2071         coords[18 * dim + 1]                = ds2;
2072         coords[18 * dim + 2]                = 0.5;
2073         /* 18 34 20 */ coords[19 * dim + 0] = -ds2;
2074         coords[19 * dim + 1]                = ds2;
2075         coords[19 * dim + 2]                = 0.5;
2076         /* 29 35 30 */ coords[20 * dim + 0] = -dis;
2077         coords[20 * dim + 1]                = -dis;
2078         coords[20 * dim + 2]                = 0.5;
2079         /* 23 36 25 */ coords[21 * dim + 0] = dis;
2080         coords[21 * dim + 1]                = -dis;
2081         coords[21 * dim + 2]                = 0.5;
2082         /* 24 37 26 */ coords[22 * dim + 0] = dis;
2083         coords[22 * dim + 1]                = dis;
2084         coords[22 * dim + 2]                = 0.5;
2085         /* 27 38 28 */ coords[23 * dim + 0] = -dis;
2086         coords[23 * dim + 1]                = dis;
2087         coords[23 * dim + 2]                = 0.5;
2088       }
2089     }
2090     PetscCall(VecRestoreArray(coordinates, &coords));
2091     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2092     PetscCall(VecDestroy(&coordinates));
2093   }
2094   /* Create periodicity */
2095   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
2096     PetscReal L[3]       = {-1., -1., 0.};
2097     PetscReal maxCell[3] = {-1., -1., 0.};
2098     PetscReal lower[3]   = {0.0, 0.0, 0.0};
2099     PetscReal upper[3]   = {1.0, 1.0, 1.5};
2100     PetscInt  numZCells  = 3;
2101 
2102     L[2]       = upper[2] - lower[2];
2103     maxCell[2] = 1.1 * (L[2] / numZCells);
2104     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
2105   }
2106   {
2107     DM          cdm;
2108     PetscDS     cds;
2109     PetscScalar c[2] = {1.0, 1.0};
2110 
2111     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_TRUE, snapToCylinder));
2112     PetscCall(DMGetCoordinateDM(dm, &cdm));
2113     PetscCall(DMGetDS(cdm, &cds));
2114     PetscCall(PetscDSSetConstants(cds, 2, c));
2115   }
2116   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
2117 
2118   /* Wait for coordinate creation before doing in-place modification */
2119   PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2120   PetscFunctionReturn(PETSC_SUCCESS);
2121 }
2122 
2123 /*@
2124   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
2125 
2126   Collective
2127 
2128   Input Parameters:
2129 + comm      - The communicator for the `DM` object
2130 - periodicZ - The boundary type for the Z direction
2131 
2132   Output Parameter:
2133 . dm - The `DM` object
2134 
2135   Level: beginner
2136 
2137   Note:
2138   Here is the output numbering looking from the bottom of the cylinder\:
2139 .vb
2140        17-----14
2141         |     |
2142         |  2  |
2143         |     |
2144  17-----8-----7-----14
2145   |     |     |     |
2146   |  3  |  0  |  1  |
2147   |     |     |     |
2148  19-----5-----6-----13
2149         |     |
2150         |  4  |
2151         |     |
2152        19-----13
2153 
2154  and up through the top
2155 
2156        18-----16
2157         |     |
2158         |  2  |
2159         |     |
2160  18----10----11-----16
2161   |     |     |     |
2162   |  3  |  0  |  1  |
2163   |     |     |     |
2164  20-----9----12-----15
2165         |     |
2166         |  4  |
2167         |     |
2168        20-----15
2169 .ve
2170 
2171 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2172 @*/
2173 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, DMBoundaryType periodicZ, DM *dm)
2174 {
2175   PetscFunctionBegin;
2176   PetscAssertPointer(dm, 3);
2177   PetscCall(DMCreate(comm, dm));
2178   PetscCall(DMSetType(*dm, DMPLEX));
2179   PetscCall(DMPlexCreateHexCylinderMesh_Internal(*dm, periodicZ));
2180   PetscFunctionReturn(PETSC_SUCCESS);
2181 }
2182 
2183 static PetscErrorCode DMPlexCreateWedgeCylinderMesh_Internal(DM dm, PetscInt n, PetscBool interpolate)
2184 {
2185   const PetscInt dim = 3;
2186   PetscInt       numCells, numVertices, v;
2187   PetscMPIInt    rank;
2188 
2189   PetscFunctionBegin;
2190   PetscCheck(n >= 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %" PetscInt_FMT " cannot be negative", n);
2191   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
2192   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2193   PetscCall(DMSetDimension(dm, dim));
2194   /* Must create the celltype label here so that we do not automatically try to compute the types */
2195   PetscCall(DMCreateLabel(dm, "celltype"));
2196   /* Create topology */
2197   {
2198     PetscInt cone[6], c;
2199 
2200     numCells    = rank == 0 ? n : 0;
2201     numVertices = rank == 0 ? 2 * (n + 1) : 0;
2202     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
2203     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
2204     PetscCall(DMSetUp(dm));
2205     for (c = 0; c < numCells; c++) {
2206       cone[0] = c + n * 1;
2207       cone[1] = (c + 1) % n + n * 1;
2208       cone[2] = 0 + 3 * n;
2209       cone[3] = c + n * 2;
2210       cone[4] = (c + 1) % n + n * 2;
2211       cone[5] = 1 + 3 * n;
2212       PetscCall(DMPlexSetCone(dm, c, cone));
2213       PetscCall(DMPlexSetCellType(dm, c, DM_POLYTOPE_TRI_PRISM_TENSOR));
2214     }
2215     PetscCall(DMPlexSymmetrize(dm));
2216     PetscCall(DMPlexStratify(dm));
2217   }
2218   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetCellType(dm, v, DM_POLYTOPE_POINT));
2219   /* Create cylinder geometry */
2220   {
2221     Vec          coordinates;
2222     PetscSection coordSection;
2223     PetscScalar *coords;
2224     PetscInt     coordSize, c;
2225 
2226     /* Build coordinates */
2227     PetscCall(DMGetCoordinateSection(dm, &coordSection));
2228     PetscCall(PetscSectionSetNumFields(coordSection, 1));
2229     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
2230     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
2231     for (v = numCells; v < numCells + numVertices; ++v) {
2232       PetscCall(PetscSectionSetDof(coordSection, v, dim));
2233       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
2234     }
2235     PetscCall(PetscSectionSetUp(coordSection));
2236     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2237     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2238     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2239     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2240     PetscCall(VecSetBlockSize(coordinates, dim));
2241     PetscCall(VecSetType(coordinates, VECSTANDARD));
2242     PetscCall(VecGetArray(coordinates, &coords));
2243     for (c = 0; c < numCells; c++) {
2244       coords[(c + 0 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
2245       coords[(c + 0 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
2246       coords[(c + 0 * n) * dim + 2] = 1.0;
2247       coords[(c + 1 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
2248       coords[(c + 1 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
2249       coords[(c + 1 * n) * dim + 2] = 0.0;
2250     }
2251     if (rank == 0) {
2252       coords[(2 * n + 0) * dim + 0] = 0.0;
2253       coords[(2 * n + 0) * dim + 1] = 0.0;
2254       coords[(2 * n + 0) * dim + 2] = 1.0;
2255       coords[(2 * n + 1) * dim + 0] = 0.0;
2256       coords[(2 * n + 1) * dim + 1] = 0.0;
2257       coords[(2 * n + 1) * dim + 2] = 0.0;
2258     }
2259     PetscCall(VecRestoreArray(coordinates, &coords));
2260     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2261     PetscCall(VecDestroy(&coordinates));
2262   }
2263   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
2264   /* Interpolate */
2265   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2266   PetscFunctionReturn(PETSC_SUCCESS);
2267 }
2268 
2269 /*@
2270   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
2271 
2272   Collective
2273 
2274   Input Parameters:
2275 + comm        - The communicator for the `DM` object
2276 . n           - The number of wedges around the origin
2277 - interpolate - Create edges and faces
2278 
2279   Output Parameter:
2280 . dm - The `DM` object
2281 
2282   Level: beginner
2283 
2284 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2285 @*/
2286 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
2287 {
2288   PetscFunctionBegin;
2289   PetscAssertPointer(dm, 4);
2290   PetscCall(DMCreate(comm, dm));
2291   PetscCall(DMSetType(*dm, DMPLEX));
2292   PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(*dm, n, interpolate));
2293   PetscFunctionReturn(PETSC_SUCCESS);
2294 }
2295 
2296 static inline PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2297 {
2298   PetscReal prod = 0.0;
2299   PetscInt  i;
2300   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
2301   return PetscSqrtReal(prod);
2302 }
2303 static inline PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2304 {
2305   PetscReal prod = 0.0;
2306   PetscInt  i;
2307   for (i = 0; i < dim; ++i) prod += x[i] * y[i];
2308   return prod;
2309 }
2310 
2311 /* The first constant is the sphere radius */
2312 static void snapToSphere(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
2313 {
2314   PetscReal r     = PetscRealPart(constants[0]);
2315   PetscReal norm2 = 0.0, fac;
2316   PetscInt  n     = uOff[1] - uOff[0], d;
2317 
2318   for (d = 0; d < n; ++d) norm2 += PetscSqr(PetscRealPart(u[d]));
2319   fac = r / PetscSqrtReal(norm2);
2320   for (d = 0; d < n; ++d) f0[d] = u[d] * fac;
2321 }
2322 
2323 static PetscErrorCode DMPlexCreateSphereMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, PetscReal R)
2324 {
2325   const PetscInt embedDim = dim + 1;
2326   PetscSection   coordSection;
2327   Vec            coordinates;
2328   PetscScalar   *coords;
2329   PetscReal     *coordsIn;
2330   PetscInt       numCells, numEdges, numVerts = 0, firstVertex = 0, v, firstEdge, coordSize, d, c, e;
2331   PetscMPIInt    rank;
2332 
2333   PetscFunctionBegin;
2334   PetscValidLogicalCollectiveBool(dm, simplex, 3);
2335   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
2336   PetscCall(DMSetDimension(dm, dim));
2337   PetscCall(DMSetCoordinateDim(dm, dim + 1));
2338   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2339   switch (dim) {
2340   case 2:
2341     if (simplex) {
2342       const PetscReal radius    = PetscSqrtReal(1 + PETSC_PHI * PETSC_PHI) / (1.0 + PETSC_PHI);
2343       const PetscReal edgeLen   = 2.0 / (1.0 + PETSC_PHI) * (R / radius);
2344       const PetscInt  degree    = 5;
2345       PetscReal       vertex[3] = {0.0, 1.0 / (1.0 + PETSC_PHI), PETSC_PHI / (1.0 + PETSC_PHI)};
2346       PetscInt        s[3]      = {1, 1, 1};
2347       PetscInt        cone[3];
2348       PetscInt       *graph, p, i, j, k;
2349 
2350       vertex[0] *= R / radius;
2351       vertex[1] *= R / radius;
2352       vertex[2] *= R / radius;
2353       numCells    = rank == 0 ? 20 : 0;
2354       numVerts    = rank == 0 ? 12 : 0;
2355       firstVertex = numCells;
2356       /* Use icosahedron, which for a R-sphere has coordinates which are all cyclic permutations of
2357 
2358            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
2359 
2360          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2361          length is then given by 2/(1+\phi) = 2 * 0.38197 = 0.76393.
2362       */
2363       /* Construct vertices */
2364       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2365       if (rank == 0) {
2366         for (p = 0, i = 0; p < embedDim; ++p) {
2367           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2368             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2369               for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertex[(d + p) % embedDim];
2370               ++i;
2371             }
2372           }
2373         }
2374       }
2375       /* Construct graph */
2376       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2377       for (i = 0; i < numVerts; ++i) {
2378         for (j = 0, k = 0; j < numVerts; ++j) {
2379           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2380             graph[i * numVerts + j] = 1;
2381             ++k;
2382           }
2383         }
2384         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid icosahedron, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2385       }
2386       /* Build Topology */
2387       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2388       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2389       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2390       /* Cells */
2391       for (i = 0, c = 0; i < numVerts; ++i) {
2392         for (j = 0; j < i; ++j) {
2393           for (k = 0; k < j; ++k) {
2394             if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i]) {
2395               cone[0] = firstVertex + i;
2396               cone[1] = firstVertex + j;
2397               cone[2] = firstVertex + k;
2398               /* Check orientation */
2399               {
2400                 const PetscInt epsilon[3][3][3] = {
2401                   {{0, 0, 0},  {0, 0, 1},  {0, -1, 0}},
2402                   {{0, 0, -1}, {0, 0, 0},  {1, 0, 0} },
2403                   {{0, 1, 0},  {-1, 0, 0}, {0, 0, 0} }
2404                 };
2405                 PetscReal normal[3];
2406                 PetscInt  e, f;
2407 
2408                 for (d = 0; d < embedDim; ++d) {
2409                   normal[d] = 0.0;
2410                   for (e = 0; e < embedDim; ++e) {
2411                     for (f = 0; f < embedDim; ++f) normal[d] += epsilon[d][e][f] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]);
2412                   }
2413                 }
2414                 if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2415                   PetscInt tmp = cone[1];
2416                   cone[1]      = cone[2];
2417                   cone[2]      = tmp;
2418                 }
2419               }
2420               PetscCall(DMPlexSetCone(dm, c++, cone));
2421             }
2422           }
2423         }
2424       }
2425       PetscCall(DMPlexSymmetrize(dm));
2426       PetscCall(DMPlexStratify(dm));
2427       PetscCall(PetscFree(graph));
2428     } else {
2429       /*
2430         12-21--13
2431          |     |
2432         25  4  24
2433          |     |
2434   12-25--9-16--8-24--13
2435    |     |     |     |
2436   23  5 17  0 15  3  22
2437    |     |     |     |
2438   10-20--6-14--7-19--11
2439          |     |
2440         20  1  19
2441          |     |
2442         10-18--11
2443          |     |
2444         23  2  22
2445          |     |
2446         12-21--13
2447        */
2448       PetscInt cone[4], ornt[4];
2449 
2450       numCells    = rank == 0 ? 6 : 0;
2451       numEdges    = rank == 0 ? 12 : 0;
2452       numVerts    = rank == 0 ? 8 : 0;
2453       firstVertex = numCells;
2454       firstEdge   = numCells + numVerts;
2455       /* Build Topology */
2456       PetscCall(DMPlexSetChart(dm, 0, numCells + numEdges + numVerts));
2457       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 4));
2458       for (e = firstEdge; e < firstEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
2459       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2460       if (rank == 0) {
2461         /* Cell 0 */
2462         cone[0] = 14;
2463         cone[1] = 15;
2464         cone[2] = 16;
2465         cone[3] = 17;
2466         PetscCall(DMPlexSetCone(dm, 0, cone));
2467         ornt[0] = 0;
2468         ornt[1] = 0;
2469         ornt[2] = 0;
2470         ornt[3] = 0;
2471         PetscCall(DMPlexSetConeOrientation(dm, 0, ornt));
2472         /* Cell 1 */
2473         cone[0] = 18;
2474         cone[1] = 19;
2475         cone[2] = 14;
2476         cone[3] = 20;
2477         PetscCall(DMPlexSetCone(dm, 1, cone));
2478         ornt[0] = 0;
2479         ornt[1] = 0;
2480         ornt[2] = -1;
2481         ornt[3] = 0;
2482         PetscCall(DMPlexSetConeOrientation(dm, 1, ornt));
2483         /* Cell 2 */
2484         cone[0] = 21;
2485         cone[1] = 22;
2486         cone[2] = 18;
2487         cone[3] = 23;
2488         PetscCall(DMPlexSetCone(dm, 2, cone));
2489         ornt[0] = 0;
2490         ornt[1] = 0;
2491         ornt[2] = -1;
2492         ornt[3] = 0;
2493         PetscCall(DMPlexSetConeOrientation(dm, 2, ornt));
2494         /* Cell 3 */
2495         cone[0] = 19;
2496         cone[1] = 22;
2497         cone[2] = 24;
2498         cone[3] = 15;
2499         PetscCall(DMPlexSetCone(dm, 3, cone));
2500         ornt[0] = -1;
2501         ornt[1] = -1;
2502         ornt[2] = 0;
2503         ornt[3] = -1;
2504         PetscCall(DMPlexSetConeOrientation(dm, 3, ornt));
2505         /* Cell 4 */
2506         cone[0] = 16;
2507         cone[1] = 24;
2508         cone[2] = 21;
2509         cone[3] = 25;
2510         PetscCall(DMPlexSetCone(dm, 4, cone));
2511         ornt[0] = -1;
2512         ornt[1] = -1;
2513         ornt[2] = -1;
2514         ornt[3] = 0;
2515         PetscCall(DMPlexSetConeOrientation(dm, 4, ornt));
2516         /* Cell 5 */
2517         cone[0] = 20;
2518         cone[1] = 17;
2519         cone[2] = 25;
2520         cone[3] = 23;
2521         PetscCall(DMPlexSetCone(dm, 5, cone));
2522         ornt[0] = -1;
2523         ornt[1] = -1;
2524         ornt[2] = -1;
2525         ornt[3] = -1;
2526         PetscCall(DMPlexSetConeOrientation(dm, 5, ornt));
2527         /* Edges */
2528         cone[0] = 6;
2529         cone[1] = 7;
2530         PetscCall(DMPlexSetCone(dm, 14, cone));
2531         cone[0] = 7;
2532         cone[1] = 8;
2533         PetscCall(DMPlexSetCone(dm, 15, cone));
2534         cone[0] = 8;
2535         cone[1] = 9;
2536         PetscCall(DMPlexSetCone(dm, 16, cone));
2537         cone[0] = 9;
2538         cone[1] = 6;
2539         PetscCall(DMPlexSetCone(dm, 17, cone));
2540         cone[0] = 10;
2541         cone[1] = 11;
2542         PetscCall(DMPlexSetCone(dm, 18, cone));
2543         cone[0] = 11;
2544         cone[1] = 7;
2545         PetscCall(DMPlexSetCone(dm, 19, cone));
2546         cone[0] = 6;
2547         cone[1] = 10;
2548         PetscCall(DMPlexSetCone(dm, 20, cone));
2549         cone[0] = 12;
2550         cone[1] = 13;
2551         PetscCall(DMPlexSetCone(dm, 21, cone));
2552         cone[0] = 13;
2553         cone[1] = 11;
2554         PetscCall(DMPlexSetCone(dm, 22, cone));
2555         cone[0] = 10;
2556         cone[1] = 12;
2557         PetscCall(DMPlexSetCone(dm, 23, cone));
2558         cone[0] = 13;
2559         cone[1] = 8;
2560         PetscCall(DMPlexSetCone(dm, 24, cone));
2561         cone[0] = 12;
2562         cone[1] = 9;
2563         PetscCall(DMPlexSetCone(dm, 25, cone));
2564       }
2565       PetscCall(DMPlexSymmetrize(dm));
2566       PetscCall(DMPlexStratify(dm));
2567       /* Build coordinates */
2568       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2569       if (rank == 0) {
2570         coordsIn[0 * embedDim + 0] = -R;
2571         coordsIn[0 * embedDim + 1] = R;
2572         coordsIn[0 * embedDim + 2] = -R;
2573         coordsIn[1 * embedDim + 0] = R;
2574         coordsIn[1 * embedDim + 1] = R;
2575         coordsIn[1 * embedDim + 2] = -R;
2576         coordsIn[2 * embedDim + 0] = R;
2577         coordsIn[2 * embedDim + 1] = -R;
2578         coordsIn[2 * embedDim + 2] = -R;
2579         coordsIn[3 * embedDim + 0] = -R;
2580         coordsIn[3 * embedDim + 1] = -R;
2581         coordsIn[3 * embedDim + 2] = -R;
2582         coordsIn[4 * embedDim + 0] = -R;
2583         coordsIn[4 * embedDim + 1] = R;
2584         coordsIn[4 * embedDim + 2] = R;
2585         coordsIn[5 * embedDim + 0] = R;
2586         coordsIn[5 * embedDim + 1] = R;
2587         coordsIn[5 * embedDim + 2] = R;
2588         coordsIn[6 * embedDim + 0] = -R;
2589         coordsIn[6 * embedDim + 1] = -R;
2590         coordsIn[6 * embedDim + 2] = R;
2591         coordsIn[7 * embedDim + 0] = R;
2592         coordsIn[7 * embedDim + 1] = -R;
2593         coordsIn[7 * embedDim + 2] = R;
2594       }
2595     }
2596     break;
2597   case 3:
2598     if (simplex) {
2599       const PetscReal edgeLen         = 1.0 / PETSC_PHI;
2600       PetscReal       vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
2601       PetscReal       vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
2602       PetscReal       vertexC[4]      = {0.5, 0.5 * PETSC_PHI, 0.5 / PETSC_PHI, 0.0};
2603       const PetscInt  degree          = 12;
2604       PetscInt        s[4]            = {1, 1, 1};
2605       PetscInt        evenPerm[12][4] = {
2606         {0, 1, 2, 3},
2607         {0, 2, 3, 1},
2608         {0, 3, 1, 2},
2609         {1, 0, 3, 2},
2610         {1, 2, 0, 3},
2611         {1, 3, 2, 0},
2612         {2, 0, 1, 3},
2613         {2, 1, 3, 0},
2614         {2, 3, 0, 1},
2615         {3, 0, 2, 1},
2616         {3, 1, 0, 2},
2617         {3, 2, 1, 0}
2618       };
2619       PetscInt  cone[4];
2620       PetscInt *graph, p, i, j, k, l;
2621 
2622       vertexA[0] *= R;
2623       vertexA[1] *= R;
2624       vertexA[2] *= R;
2625       vertexA[3] *= R;
2626       vertexB[0] *= R;
2627       vertexB[1] *= R;
2628       vertexB[2] *= R;
2629       vertexB[3] *= R;
2630       vertexC[0] *= R;
2631       vertexC[1] *= R;
2632       vertexC[2] *= R;
2633       vertexC[3] *= R;
2634       numCells    = rank == 0 ? 600 : 0;
2635       numVerts    = rank == 0 ? 120 : 0;
2636       firstVertex = numCells;
2637       /* Use the 600-cell, which for a unit sphere has coordinates which are
2638 
2639            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
2640                (\pm 1,    0,       0,      0)  all cyclic permutations   8
2641            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
2642 
2643          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2644          length is then given by 1/\phi = 0.61803.
2645 
2646          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
2647          http://mathworld.wolfram.com/600-Cell.html
2648       */
2649       /* Construct vertices */
2650       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2651       i = 0;
2652       if (rank == 0) {
2653         for (s[0] = -1; s[0] < 2; s[0] += 2) {
2654           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2655             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2656               for (s[3] = -1; s[3] < 2; s[3] += 2) {
2657                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[d] * vertexA[d];
2658                 ++i;
2659               }
2660             }
2661           }
2662         }
2663         for (p = 0; p < embedDim; ++p) {
2664           s[1] = s[2] = s[3] = 1;
2665           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2666             for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertexB[(d + p) % embedDim];
2667             ++i;
2668           }
2669         }
2670         for (p = 0; p < 12; ++p) {
2671           s[3] = 1;
2672           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2673             for (s[1] = -1; s[1] < 2; s[1] += 2) {
2674               for (s[2] = -1; s[2] < 2; s[2] += 2) {
2675                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[evenPerm[p][d]] * vertexC[evenPerm[p][d]];
2676                 ++i;
2677               }
2678             }
2679           }
2680         }
2681       }
2682       PetscCheck(i == numVerts, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertices %" PetscInt_FMT " != %" PetscInt_FMT, i, numVerts);
2683       /* Construct graph */
2684       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2685       for (i = 0; i < numVerts; ++i) {
2686         for (j = 0, k = 0; j < numVerts; ++j) {
2687           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2688             graph[i * numVerts + j] = 1;
2689             ++k;
2690           }
2691         }
2692         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2693       }
2694       /* Build Topology */
2695       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2696       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2697       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2698       /* Cells */
2699       if (rank == 0) {
2700         for (i = 0, c = 0; i < numVerts; ++i) {
2701           for (j = 0; j < i; ++j) {
2702             for (k = 0; k < j; ++k) {
2703               for (l = 0; l < k; ++l) {
2704                 if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i] && graph[l * numVerts + i] && graph[l * numVerts + j] && graph[l * numVerts + k]) {
2705                   cone[0] = firstVertex + i;
2706                   cone[1] = firstVertex + j;
2707                   cone[2] = firstVertex + k;
2708                   cone[3] = firstVertex + l;
2709                   /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
2710                   {
2711                     const PetscInt epsilon[4][4][4][4] = {
2712                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, -1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 0, 0}, {0, 1, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 1, 0}, {0, -1, 0, 0}, {0, 0, 0, 0}}},
2713 
2714                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}}, {{0, 0, -1, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}}},
2715 
2716                       {{{0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, 0, 0}, {0, -1, 0, 0}}, {{0, 0, 0, -1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 1, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}},
2717 
2718                       {{{0, 0, 0, 0}, {0, 0, -1, 0}, {0, 1, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 1, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}}, {{0, -1, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}} }
2719                     };
2720                     PetscReal normal[4];
2721                     PetscInt  e, f, g;
2722 
2723                     for (d = 0; d < embedDim; ++d) {
2724                       normal[d] = 0.0;
2725                       for (e = 0; e < embedDim; ++e) {
2726                         for (f = 0; f < embedDim; ++f) {
2727                           for (g = 0; g < embedDim; ++g) {
2728                             normal[d] += epsilon[d][e][f][g] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]) * (coordsIn[l * embedDim + f] - coordsIn[i * embedDim + f]);
2729                           }
2730                         }
2731                       }
2732                     }
2733                     if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2734                       PetscInt tmp = cone[1];
2735                       cone[1]      = cone[2];
2736                       cone[2]      = tmp;
2737                     }
2738                   }
2739                   PetscCall(DMPlexSetCone(dm, c++, cone));
2740                 }
2741               }
2742             }
2743           }
2744         }
2745       }
2746       PetscCall(DMPlexSymmetrize(dm));
2747       PetscCall(DMPlexStratify(dm));
2748       PetscCall(PetscFree(graph));
2749     }
2750     break;
2751   default:
2752     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unsupported dimension for sphere: %" PetscInt_FMT, dim);
2753   }
2754   /* Create coordinates */
2755   PetscCall(DMGetCoordinateSection(dm, &coordSection));
2756   PetscCall(PetscSectionSetNumFields(coordSection, 1));
2757   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, embedDim));
2758   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVerts));
2759   for (v = firstVertex; v < firstVertex + numVerts; ++v) {
2760     PetscCall(PetscSectionSetDof(coordSection, v, embedDim));
2761     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, embedDim));
2762   }
2763   PetscCall(PetscSectionSetUp(coordSection));
2764   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2765   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2766   PetscCall(VecSetBlockSize(coordinates, embedDim));
2767   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2768   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2769   PetscCall(VecSetType(coordinates, VECSTANDARD));
2770   PetscCall(VecGetArray(coordinates, &coords));
2771   for (v = 0; v < numVerts; ++v)
2772     for (d = 0; d < embedDim; ++d) coords[v * embedDim + d] = coordsIn[v * embedDim + d];
2773   PetscCall(VecRestoreArray(coordinates, &coords));
2774   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2775   PetscCall(VecDestroy(&coordinates));
2776   PetscCall(PetscFree(coordsIn));
2777   {
2778     DM          cdm;
2779     PetscDS     cds;
2780     PetscScalar c = R;
2781 
2782     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_TRUE, snapToSphere));
2783     PetscCall(DMGetCoordinateDM(dm, &cdm));
2784     PetscCall(DMGetDS(cdm, &cds));
2785     PetscCall(PetscDSSetConstants(cds, 1, &c));
2786   }
2787   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
2788   /* Wait for coordinate creation before doing in-place modification */
2789   if (simplex) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2790   PetscFunctionReturn(PETSC_SUCCESS);
2791 }
2792 
2793 typedef void (*TPSEvaluateFunc)(const PetscReal[], PetscReal *, PetscReal[], PetscReal (*)[3]);
2794 
2795 /*
2796  The Schwarz P implicit surface is
2797 
2798      f(x) = cos(x0) + cos(x1) + cos(x2) = 0
2799 */
2800 static void TPSEvaluate_SchwarzP(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2801 {
2802   PetscReal c[3] = {PetscCosReal(y[0] * PETSC_PI), PetscCosReal(y[1] * PETSC_PI), PetscCosReal(y[2] * PETSC_PI)};
2803   PetscReal g[3] = {-PetscSinReal(y[0] * PETSC_PI), -PetscSinReal(y[1] * PETSC_PI), -PetscSinReal(y[2] * PETSC_PI)};
2804   f[0]           = c[0] + c[1] + c[2];
2805   for (PetscInt i = 0; i < 3; i++) {
2806     grad[i] = PETSC_PI * g[i];
2807     for (PetscInt j = 0; j < 3; j++) hess[i][j] = (i == j) ? -PetscSqr(PETSC_PI) * c[i] : 0.;
2808   }
2809 }
2810 
2811 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2812 static PetscErrorCode TPSExtrudeNormalFunc_SchwarzP(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2813 {
2814   for (PetscInt i = 0; i < 3; i++) u[i] = -PETSC_PI * PetscSinReal(x[i] * PETSC_PI);
2815   return PETSC_SUCCESS;
2816 }
2817 
2818 /*
2819  The Gyroid implicit surface is
2820 
2821  f(x,y,z) = sin(pi * x) * cos (pi * (y + 1/2))  + sin(pi * (y + 1/2)) * cos(pi * (z + 1/4)) + sin(pi * (z + 1/4)) * cos(pi * x)
2822 
2823 */
2824 static void TPSEvaluate_Gyroid(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2825 {
2826   PetscReal s[3] = {PetscSinReal(PETSC_PI * y[0]), PetscSinReal(PETSC_PI * (y[1] + .5)), PetscSinReal(PETSC_PI * (y[2] + .25))};
2827   PetscReal c[3] = {PetscCosReal(PETSC_PI * y[0]), PetscCosReal(PETSC_PI * (y[1] + .5)), PetscCosReal(PETSC_PI * (y[2] + .25))};
2828   f[0]           = s[0] * c[1] + s[1] * c[2] + s[2] * c[0];
2829   grad[0]        = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2830   grad[1]        = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2831   grad[2]        = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2832   hess[0][0]     = -PetscSqr(PETSC_PI) * (s[0] * c[1] + s[2] * c[0]);
2833   hess[0][1]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2834   hess[0][2]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2835   hess[1][0]     = -PetscSqr(PETSC_PI) * (s[1] * c[2] + s[0] * c[1]);
2836   hess[1][1]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2837   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2838   hess[2][0]     = -PetscSqr(PETSC_PI) * (s[2] * c[0] + s[1] * c[2]);
2839   hess[2][1]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2840   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2841 }
2842 
2843 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2844 static PetscErrorCode TPSExtrudeNormalFunc_Gyroid(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2845 {
2846   PetscReal s[3] = {PetscSinReal(PETSC_PI * x[0]), PetscSinReal(PETSC_PI * (x[1] + .5)), PetscSinReal(PETSC_PI * (x[2] + .25))};
2847   PetscReal c[3] = {PetscCosReal(PETSC_PI * x[0]), PetscCosReal(PETSC_PI * (x[1] + .5)), PetscCosReal(PETSC_PI * (x[2] + .25))};
2848   u[0]           = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2849   u[1]           = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2850   u[2]           = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2851   return PETSC_SUCCESS;
2852 }
2853 
2854 /*
2855    We wish to solve
2856 
2857          min_y || y - x ||^2  subject to f(y) = 0
2858 
2859    Let g(y) = grad(f).  The minimization problem is equivalent to asking to satisfy
2860    f(y) = 0 and (y-x) is parallel to g(y).  We do this by using Householder QR to obtain a basis for the
2861    tangent space and ask for both components in the tangent space to be zero.
2862 
2863    Take g to be a column vector and compute the "full QR" factorization Q R = g,
2864    where Q = I - 2 n n^T is a symmetric orthogonal matrix.
2865    The first column of Q is parallel to g so the remaining two columns span the null space.
2866    Let Qn = Q[:,1:] be those remaining columns.  Then Qn Qn^T is an orthogonal projector into the tangent space.
2867    Since Q is symmetric, this is equivalent to multiplying by Q and taking the last two entries.
2868    In total, we have a system of 3 equations in 3 unknowns:
2869 
2870      f(y) = 0                       1 equation
2871      Qn^T (y - x) = 0               2 equations
2872 
2873    Here, we compute the residual and Jacobian of this system.
2874 */
2875 static void TPSNearestPointResJac(TPSEvaluateFunc feval, const PetscScalar x[], const PetscScalar y[], PetscScalar res[], PetscScalar J[])
2876 {
2877   PetscReal yreal[3] = {PetscRealPart(y[0]), PetscRealPart(y[1]), PetscRealPart(y[2])};
2878   PetscReal d[3]     = {PetscRealPart(y[0] - x[0]), PetscRealPart(y[1] - x[1]), PetscRealPart(y[2] - x[2])};
2879   PetscReal f, grad[3], n[3], norm, norm_y[3], nd, nd_y[3], sign;
2880   PetscReal n_y[3][3] = {
2881     {0, 0, 0},
2882     {0, 0, 0},
2883     {0, 0, 0}
2884   };
2885 
2886   feval(yreal, &f, grad, n_y);
2887 
2888   for (PetscInt i = 0; i < 3; i++) n[i] = grad[i];
2889   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2890   for (PetscInt i = 0; i < 3; i++) norm_y[i] = 1. / norm * n[i] * n_y[i][i];
2891 
2892   // Define the Householder reflector
2893   sign = n[0] >= 0 ? 1. : -1.;
2894   n[0] += norm * sign;
2895   for (PetscInt i = 0; i < 3; i++) n_y[0][i] += norm_y[i] * sign;
2896 
2897   norm      = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2898   norm_y[0] = 1. / norm * (n[0] * n_y[0][0]);
2899   norm_y[1] = 1. / norm * (n[0] * n_y[0][1] + n[1] * n_y[1][1]);
2900   norm_y[2] = 1. / norm * (n[0] * n_y[0][2] + n[2] * n_y[2][2]);
2901 
2902   for (PetscInt i = 0; i < 3; i++) {
2903     n[i] /= norm;
2904     for (PetscInt j = 0; j < 3; j++) {
2905       // note that n[i] is n_old[i]/norm when executing the code below
2906       n_y[i][j] = n_y[i][j] / norm - n[i] / norm * norm_y[j];
2907     }
2908   }
2909 
2910   nd = n[0] * d[0] + n[1] * d[1] + n[2] * d[2];
2911   for (PetscInt i = 0; i < 3; i++) nd_y[i] = n[i] + n_y[0][i] * d[0] + n_y[1][i] * d[1] + n_y[2][i] * d[2];
2912 
2913   res[0] = f;
2914   res[1] = d[1] - 2 * n[1] * nd;
2915   res[2] = d[2] - 2 * n[2] * nd;
2916   // J[j][i] is J_{ij} (column major)
2917   for (PetscInt j = 0; j < 3; j++) {
2918     J[0 + j * 3] = grad[j];
2919     J[1 + j * 3] = (j == 1) * 1. - 2 * (n_y[1][j] * nd + n[1] * nd_y[j]);
2920     J[2 + j * 3] = (j == 2) * 1. - 2 * (n_y[2][j] * nd + n[2] * nd_y[j]);
2921   }
2922 }
2923 
2924 /*
2925    Project x to the nearest point on the implicit surface using Newton's method.
2926 */
2927 static PetscErrorCode TPSNearestPoint(TPSEvaluateFunc feval, PetscScalar x[])
2928 {
2929   PetscScalar y[3] = {x[0], x[1], x[2]}; // Initial guess
2930 
2931   PetscFunctionBegin;
2932   for (PetscInt iter = 0; iter < 10; iter++) {
2933     PetscScalar res[3], J[9];
2934     PetscReal   resnorm;
2935     TPSNearestPointResJac(feval, x, y, res, J);
2936     resnorm = PetscSqrtReal(PetscSqr(PetscRealPart(res[0])) + PetscSqr(PetscRealPart(res[1])) + PetscSqr(PetscRealPart(res[2])));
2937     if (0) { // Turn on this monitor if you need to confirm quadratic convergence
2938       PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%" PetscInt_FMT "] res [%g %g %g]\n", iter, (double)PetscRealPart(res[0]), (double)PetscRealPart(res[1]), (double)PetscRealPart(res[2])));
2939     }
2940     if (resnorm < PETSC_SMALL) break;
2941 
2942     // Take the Newton step
2943     PetscCall(PetscKernel_A_gets_inverse_A_3(J, 0., PETSC_FALSE, NULL));
2944     PetscKernel_v_gets_v_minus_A_times_w_3(y, J, res);
2945   }
2946   for (PetscInt i = 0; i < 3; i++) x[i] = y[i];
2947   PetscFunctionReturn(PETSC_SUCCESS);
2948 }
2949 
2950 const char *const DMPlexTPSTypes[] = {"SCHWARZ_P", "GYROID", "DMPlexTPSType", "DMPLEX_TPS_", NULL};
2951 
2952 static PetscErrorCode DMPlexCreateTPSMesh_Internal(DM dm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness)
2953 {
2954   PetscMPIInt rank;
2955   PetscInt    topoDim = 2, spaceDim = 3, numFaces = 0, numVertices = 0, numEdges = 0;
2956   PetscInt(*edges)[2] = NULL, *edgeSets = NULL;
2957   PetscInt            *cells_flat = NULL;
2958   PetscReal           *vtxCoords  = NULL;
2959   TPSEvaluateFunc      evalFunc   = NULL;
2960   PetscSimplePointFunc normalFunc = NULL;
2961   DMLabel              label;
2962 
2963   PetscFunctionBegin;
2964   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
2965   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2966   PetscCheck((layers != 0) ^ (thickness == 0.), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Layers %" PetscInt_FMT " must be nonzero iff thickness %g is nonzero", layers, (double)thickness);
2967   switch (tpstype) {
2968   case DMPLEX_TPS_SCHWARZ_P:
2969     PetscCheck(!periodic || (periodic[0] == DM_BOUNDARY_NONE && periodic[1] == DM_BOUNDARY_NONE && periodic[2] == DM_BOUNDARY_NONE), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Schwarz P does not support periodic meshes");
2970     if (rank == 0) {
2971       PetscInt(*cells)[6][4][4] = NULL; // [junction, junction-face, cell, conn]
2972       PetscInt  Njunctions = 0, Ncuts = 0, Npipes[3], vcount;
2973       PetscReal L = 1;
2974 
2975       Npipes[0]   = (extent[0] + 1) * extent[1] * extent[2];
2976       Npipes[1]   = extent[0] * (extent[1] + 1) * extent[2];
2977       Npipes[2]   = extent[0] * extent[1] * (extent[2] + 1);
2978       Njunctions  = extent[0] * extent[1] * extent[2];
2979       Ncuts       = 2 * (extent[0] * extent[1] + extent[1] * extent[2] + extent[2] * extent[0]);
2980       numVertices = 4 * (Npipes[0] + Npipes[1] + Npipes[2]) + 8 * Njunctions;
2981       PetscCall(PetscMalloc1(3 * numVertices, &vtxCoords));
2982       PetscCall(PetscMalloc1(Njunctions, &cells));
2983       PetscCall(PetscMalloc1(Ncuts * 4, &edges));
2984       PetscCall(PetscMalloc1(Ncuts * 4, &edgeSets));
2985       // x-normal pipes
2986       vcount = 0;
2987       for (PetscInt i = 0; i < extent[0] + 1; i++) {
2988         for (PetscInt j = 0; j < extent[1]; j++) {
2989           for (PetscInt k = 0; k < extent[2]; k++) {
2990             for (PetscInt l = 0; l < 4; l++) {
2991               vtxCoords[vcount++] = (2 * i - 1) * L;
2992               vtxCoords[vcount++] = 2 * j * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2993               vtxCoords[vcount++] = 2 * k * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2994             }
2995           }
2996         }
2997       }
2998       // y-normal pipes
2999       for (PetscInt i = 0; i < extent[0]; i++) {
3000         for (PetscInt j = 0; j < extent[1] + 1; j++) {
3001           for (PetscInt k = 0; k < extent[2]; k++) {
3002             for (PetscInt l = 0; l < 4; l++) {
3003               vtxCoords[vcount++] = 2 * i * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3004               vtxCoords[vcount++] = (2 * j - 1) * L;
3005               vtxCoords[vcount++] = 2 * k * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3006             }
3007           }
3008         }
3009       }
3010       // z-normal pipes
3011       for (PetscInt i = 0; i < extent[0]; i++) {
3012         for (PetscInt j = 0; j < extent[1]; j++) {
3013           for (PetscInt k = 0; k < extent[2] + 1; k++) {
3014             for (PetscInt l = 0; l < 4; l++) {
3015               vtxCoords[vcount++] = 2 * i * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3016               vtxCoords[vcount++] = 2 * j * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3017               vtxCoords[vcount++] = (2 * k - 1) * L;
3018             }
3019           }
3020         }
3021       }
3022       // junctions
3023       for (PetscInt i = 0; i < extent[0]; i++) {
3024         for (PetscInt j = 0; j < extent[1]; j++) {
3025           for (PetscInt k = 0; k < extent[2]; k++) {
3026             const PetscInt J = (i * extent[1] + j) * extent[2] + k, Jvoff = (Npipes[0] + Npipes[1] + Npipes[2]) * 4 + J * 8;
3027             PetscCheck(vcount / 3 == Jvoff, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected vertex count");
3028             for (PetscInt ii = 0; ii < 2; ii++) {
3029               for (PetscInt jj = 0; jj < 2; jj++) {
3030                 for (PetscInt kk = 0; kk < 2; kk++) {
3031                   double Ls           = (1 - sqrt(2) / 4) * L;
3032                   vtxCoords[vcount++] = 2 * i * L + (2 * ii - 1) * Ls;
3033                   vtxCoords[vcount++] = 2 * j * L + (2 * jj - 1) * Ls;
3034                   vtxCoords[vcount++] = 2 * k * L + (2 * kk - 1) * Ls;
3035                 }
3036               }
3037             }
3038             const PetscInt jfaces[3][2][4] = {
3039               {{3, 1, 0, 2}, {7, 5, 4, 6}}, // x-aligned
3040               {{5, 4, 0, 1}, {7, 6, 2, 3}}, // y-aligned
3041               {{6, 2, 0, 4}, {7, 3, 1, 5}}  // z-aligned
3042             };
3043             const PetscInt pipe_lo[3] = {// vertex numbers of pipes
3044                                          ((i * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + Npipes[0] + Npipes[1]) * 4};
3045             const PetscInt pipe_hi[3] = {// vertex numbers of pipes
3046                                          (((i + 1) * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j + 1) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + 1 + Npipes[0] + Npipes[1]) * 4};
3047             for (PetscInt dir = 0; dir < 3; dir++) { // x,y,z
3048               const PetscInt ijk[3] = {i, j, k};
3049               for (PetscInt l = 0; l < 4; l++) { // rotations
3050                 cells[J][dir * 2 + 0][l][0] = pipe_lo[dir] + l;
3051                 cells[J][dir * 2 + 0][l][1] = Jvoff + jfaces[dir][0][l];
3052                 cells[J][dir * 2 + 0][l][2] = Jvoff + jfaces[dir][0][(l - 1 + 4) % 4];
3053                 cells[J][dir * 2 + 0][l][3] = pipe_lo[dir] + (l - 1 + 4) % 4;
3054                 cells[J][dir * 2 + 1][l][0] = Jvoff + jfaces[dir][1][l];
3055                 cells[J][dir * 2 + 1][l][1] = pipe_hi[dir] + l;
3056                 cells[J][dir * 2 + 1][l][2] = pipe_hi[dir] + (l - 1 + 4) % 4;
3057                 cells[J][dir * 2 + 1][l][3] = Jvoff + jfaces[dir][1][(l - 1 + 4) % 4];
3058                 if (ijk[dir] == 0) {
3059                   edges[numEdges][0] = pipe_lo[dir] + l;
3060                   edges[numEdges][1] = pipe_lo[dir] + (l + 1) % 4;
3061                   edgeSets[numEdges] = dir * 2 + 1;
3062                   numEdges++;
3063                 }
3064                 if (ijk[dir] + 1 == extent[dir]) {
3065                   edges[numEdges][0] = pipe_hi[dir] + l;
3066                   edges[numEdges][1] = pipe_hi[dir] + (l + 1) % 4;
3067                   edgeSets[numEdges] = dir * 2 + 2;
3068                   numEdges++;
3069                 }
3070               }
3071             }
3072           }
3073         }
3074       }
3075       PetscCheck(numEdges == Ncuts * 4, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Edge count %" PetscInt_FMT " incompatible with number of cuts %" PetscInt_FMT, numEdges, Ncuts);
3076       numFaces   = 24 * Njunctions;
3077       cells_flat = cells[0][0][0];
3078     }
3079     evalFunc   = TPSEvaluate_SchwarzP;
3080     normalFunc = TPSExtrudeNormalFunc_SchwarzP;
3081     break;
3082   case DMPLEX_TPS_GYROID:
3083     if (rank == 0) {
3084       // This is a coarse mesh approximation of the gyroid shifted to being the zero of the level set
3085       //
3086       //     sin(pi*x)*cos(pi*(y+1/2)) + sin(pi*(y+1/2))*cos(pi*(z+1/4)) + sin(pi*(z+1/4))*cos(x)
3087       //
3088       // on the cell [0,2]^3.
3089       //
3090       // Think about dividing that cell into four columns, and focus on the column [0,1]x[0,1]x[0,2].
3091       // If you looked at the gyroid in that column at different slices of z you would see that it kind of spins
3092       // like a boomerang:
3093       //
3094       //     z = 0          z = 1/4        z = 1/2        z = 3/4     //
3095       //     -----          -------        -------        -------     //
3096       //                                                              //
3097       //     +       +      +       +      +       +      +   \   +   //
3098       //      \                                   /            \      //
3099       //       \            `-_   _-'            /              }     //
3100       //        *-_            `-'            _-'              /      //
3101       //     +     `-+      +       +      +-'     +      +   /   +   //
3102       //                                                              //
3103       //                                                              //
3104       //     z = 1          z = 5/4        z = 3/2        z = 7/4     //
3105       //     -----          -------        -------        -------     //
3106       //                                                              //
3107       //     +-_     +      +       +      +     _-+      +   /   +   //
3108       //        `-_            _-_            _-`            /        //
3109       //           \        _-'   `-_        /              {         //
3110       //            \                       /                \        //
3111       //     +       +      +       +      +       +      +   \   +   //
3112       //
3113       //
3114       // This course mesh approximates each of these slices by two line segments,
3115       // and then connects the segments in consecutive layers with quadrilateral faces.
3116       // All of the end points of the segments are multiples of 1/4 except for the
3117       // point * in the picture for z = 0 above and the similar points in other layers.
3118       // That point is at (gamma, gamma, 0), where gamma is calculated below.
3119       //
3120       // The column  [1,2]x[1,2]x[0,2] looks the same as this column;
3121       // The columns [1,2]x[0,1]x[0,2] and [0,1]x[1,2]x[0,2] are mirror images.
3122       //
3123       // As for how this method turned into the names given to the vertices:
3124       // that was not systematic, it was just the way it worked out in my handwritten notes.
3125 
3126       PetscInt facesPerBlock = 64;
3127       PetscInt vertsPerBlock = 56;
3128       PetscInt extentPlus[3];
3129       PetscInt numBlocks, numBlocksPlus;
3130       const PetscInt A = 0, B = 1, C = 2, D = 3, E = 4, F = 5, G = 6, H = 7, II = 8, J = 9, K = 10, L = 11, M = 12, N = 13, O = 14, P = 15, Q = 16, R = 17, S = 18, T = 19, U = 20, V = 21, W = 22, X = 23, Y = 24, Z = 25, Ap = 26, Bp = 27, Cp = 28, Dp = 29, Ep = 30, Fp = 31, Gp = 32, Hp = 33, Ip = 34, Jp = 35, Kp = 36, Lp = 37, Mp = 38, Np = 39, Op = 40, Pp = 41, Qp = 42, Rp = 43, Sp = 44, Tp = 45, Up = 46, Vp = 47, Wp = 48, Xp = 49, Yp = 50, Zp = 51, Aq = 52, Bq = 53, Cq = 54, Dq = 55;
3131       const PetscInt pattern[64][4] = {
3132   /* face to vertex within the coarse discretization of a single gyroid block */
3133   /* layer 0 */
3134         {A,           C,           K,           G          },
3135         {C,           B,           II,          K          },
3136         {D,           A,           H,           L          },
3137         {B + 56 * 1,  D,           L,           J          },
3138         {E,           B + 56 * 1,  J,           N          },
3139         {A + 56 * 2,  E,           N,           H + 56 * 2 },
3140         {F,           A + 56 * 2,  G + 56 * 2,  M          },
3141         {B,           F,           M,           II         },
3142  /* layer 1 */
3143         {G,           K,           Q,           O          },
3144         {K,           II,          P,           Q          },
3145         {L,           H,           O + 56 * 1,  R          },
3146         {J,           L,           R,           P          },
3147         {N,           J,           P,           S          },
3148         {H + 56 * 2,  N,           S,           O + 56 * 3 },
3149         {M,           G + 56 * 2,  O + 56 * 2,  T          },
3150         {II,          M,           T,           P          },
3151  /* layer 2 */
3152         {O,           Q,           Y,           U          },
3153         {Q,           P,           W,           Y          },
3154         {R,           O + 56 * 1,  U + 56 * 1,  Ap         },
3155         {P,           R,           Ap,          W          },
3156         {S,           P,           X,           Bp         },
3157         {O + 56 * 3,  S,           Bp,          V + 56 * 1 },
3158         {T,           O + 56 * 2,  V,           Z          },
3159         {P,           T,           Z,           X          },
3160  /* layer 3 */
3161         {U,           Y,           Ep,          Dp         },
3162         {Y,           W,           Cp,          Ep         },
3163         {Ap,          U + 56 * 1,  Dp + 56 * 1, Gp         },
3164         {W,           Ap,          Gp,          Cp         },
3165         {Bp,          X,           Cp + 56 * 2, Fp         },
3166         {V + 56 * 1,  Bp,          Fp,          Dp + 56 * 1},
3167         {Z,           V,           Dp,          Hp         },
3168         {X,           Z,           Hp,          Cp + 56 * 2},
3169  /* layer 4 */
3170         {Dp,          Ep,          Mp,          Kp         },
3171         {Ep,          Cp,          Ip,          Mp         },
3172         {Gp,          Dp + 56 * 1, Lp,          Np         },
3173         {Cp,          Gp,          Np,          Jp         },
3174         {Fp,          Cp + 56 * 2, Jp + 56 * 2, Pp         },
3175         {Dp + 56 * 1, Fp,          Pp,          Lp         },
3176         {Hp,          Dp,          Kp,          Op         },
3177         {Cp + 56 * 2, Hp,          Op,          Ip + 56 * 2},
3178  /* layer 5 */
3179         {Kp,          Mp,          Sp,          Rp         },
3180         {Mp,          Ip,          Qp,          Sp         },
3181         {Np,          Lp,          Rp,          Tp         },
3182         {Jp,          Np,          Tp,          Qp + 56 * 1},
3183         {Pp,          Jp + 56 * 2, Qp + 56 * 3, Up         },
3184         {Lp,          Pp,          Up,          Rp         },
3185         {Op,          Kp,          Rp,          Vp         },
3186         {Ip + 56 * 2, Op,          Vp,          Qp + 56 * 2},
3187  /* layer 6 */
3188         {Rp,          Sp,          Aq,          Yp         },
3189         {Sp,          Qp,          Wp,          Aq         },
3190         {Tp,          Rp,          Yp,          Cq         },
3191         {Qp + 56 * 1, Tp,          Cq,          Wp + 56 * 1},
3192         {Up,          Qp + 56 * 3, Xp + 56 * 1, Dq         },
3193         {Rp,          Up,          Dq,          Zp         },
3194         {Vp,          Rp,          Zp,          Bq         },
3195         {Qp + 56 * 2, Vp,          Bq,          Xp         },
3196  /* layer 7 (the top is the periodic image of the bottom of layer 0) */
3197         {Yp,          Aq,          C + 56 * 4,  A + 56 * 4 },
3198         {Aq,          Wp,          B + 56 * 4,  C + 56 * 4 },
3199         {Cq,          Yp,          A + 56 * 4,  D + 56 * 4 },
3200         {Wp + 56 * 1, Cq,          D + 56 * 4,  B + 56 * 5 },
3201         {Dq,          Xp + 56 * 1, B + 56 * 5,  E + 56 * 4 },
3202         {Zp,          Dq,          E + 56 * 4,  A + 56 * 6 },
3203         {Bq,          Zp,          A + 56 * 6,  F + 56 * 4 },
3204         {Xp,          Bq,          F + 56 * 4,  B + 56 * 4 }
3205       };
3206       const PetscReal gamma                = PetscAcosReal((PetscSqrtReal(3.) - 1.) / PetscSqrtReal(2.)) / PETSC_PI;
3207       const PetscReal patternCoords[56][3] = {
3208         {1.,        0.,        0.  }, /* A  */
3209         {0.,        1.,        0.  }, /* B  */
3210         {gamma,     gamma,     0.  }, /* C  */
3211         {1 + gamma, 1 - gamma, 0.  }, /* D  */
3212         {2 - gamma, 2 - gamma, 0.  }, /* E  */
3213         {1 - gamma, 1 + gamma, 0.  }, /* F  */
3214 
3215         {.5,        0,         .25 }, /* G  */
3216         {1.5,       0.,        .25 }, /* H  */
3217         {.5,        1.,        .25 }, /* II */
3218         {1.5,       1.,        .25 }, /* J  */
3219         {.25,       .5,        .25 }, /* K  */
3220         {1.25,      .5,        .25 }, /* L  */
3221         {.75,       1.5,       .25 }, /* M  */
3222         {1.75,      1.5,       .25 }, /* N  */
3223 
3224         {0.,        0.,        .5  }, /* O  */
3225         {1.,        1.,        .5  }, /* P  */
3226         {gamma,     1 - gamma, .5  }, /* Q  */
3227         {1 + gamma, gamma,     .5  }, /* R  */
3228         {2 - gamma, 1 + gamma, .5  }, /* S  */
3229         {1 - gamma, 2 - gamma, .5  }, /* T  */
3230 
3231         {0.,        .5,        .75 }, /* U  */
3232         {0.,        1.5,       .75 }, /* V  */
3233         {1.,        .5,        .75 }, /* W  */
3234         {1.,        1.5,       .75 }, /* X  */
3235         {.5,        .75,       .75 }, /* Y  */
3236         {.5,        1.75,      .75 }, /* Z  */
3237         {1.5,       .25,       .75 }, /* Ap */
3238         {1.5,       1.25,      .75 }, /* Bp */
3239 
3240         {1.,        0.,        1.  }, /* Cp */
3241         {0.,        1.,        1.  }, /* Dp */
3242         {1 - gamma, 1 - gamma, 1.  }, /* Ep */
3243         {1 + gamma, 1 + gamma, 1.  }, /* Fp */
3244         {2 - gamma, gamma,     1.  }, /* Gp */
3245         {gamma,     2 - gamma, 1.  }, /* Hp */
3246 
3247         {.5,        0.,        1.25}, /* Ip */
3248         {1.5,       0.,        1.25}, /* Jp */
3249         {.5,        1.,        1.25}, /* Kp */
3250         {1.5,       1.,        1.25}, /* Lp */
3251         {.75,       .5,        1.25}, /* Mp */
3252         {1.75,      .5,        1.25}, /* Np */
3253         {.25,       1.5,       1.25}, /* Op */
3254         {1.25,      1.5,       1.25}, /* Pp */
3255 
3256         {0.,        0.,        1.5 }, /* Qp */
3257         {1.,        1.,        1.5 }, /* Rp */
3258         {1 - gamma, gamma,     1.5 }, /* Sp */
3259         {2 - gamma, 1 - gamma, 1.5 }, /* Tp */
3260         {1 + gamma, 2 - gamma, 1.5 }, /* Up */
3261         {gamma,     1 + gamma, 1.5 }, /* Vp */
3262 
3263         {0.,        .5,        1.75}, /* Wp */
3264         {0.,        1.5,       1.75}, /* Xp */
3265         {1.,        .5,        1.75}, /* Yp */
3266         {1.,        1.5,       1.75}, /* Zp */
3267         {.5,        .25,       1.75}, /* Aq */
3268         {.5,        1.25,      1.75}, /* Bq */
3269         {1.5,       .75,       1.75}, /* Cq */
3270         {1.5,       1.75,      1.75}, /* Dq */
3271       };
3272       PetscInt(*cells)[64][4] = NULL;
3273       PetscBool *seen;
3274       PetscInt  *vertToTrueVert;
3275       PetscInt   count;
3276 
3277       for (PetscInt i = 0; i < 3; i++) extentPlus[i] = extent[i] + 1;
3278       numBlocks = 1;
3279       for (PetscInt i = 0; i < 3; i++) numBlocks *= extent[i];
3280       numBlocksPlus = 1;
3281       for (PetscInt i = 0; i < 3; i++) numBlocksPlus *= extentPlus[i];
3282       numFaces = numBlocks * facesPerBlock;
3283       PetscCall(PetscMalloc1(numBlocks, &cells));
3284       PetscCall(PetscCalloc1(numBlocksPlus * vertsPerBlock, &seen));
3285       for (PetscInt k = 0; k < extent[2]; k++) {
3286         for (PetscInt j = 0; j < extent[1]; j++) {
3287           for (PetscInt i = 0; i < extent[0]; i++) {
3288             for (PetscInt f = 0; f < facesPerBlock; f++) {
3289               for (PetscInt v = 0; v < 4; v++) {
3290                 PetscInt vertRaw     = pattern[f][v];
3291                 PetscInt blockidx    = vertRaw / 56;
3292                 PetscInt patternvert = vertRaw % 56;
3293                 PetscInt xplus       = (blockidx & 1);
3294                 PetscInt yplus       = (blockidx & 2) >> 1;
3295                 PetscInt zplus       = (blockidx & 4) >> 2;
3296                 PetscInt zcoord      = (periodic && periodic[2] == DM_BOUNDARY_PERIODIC) ? ((k + zplus) % extent[2]) : (k + zplus);
3297                 PetscInt ycoord      = (periodic && periodic[1] == DM_BOUNDARY_PERIODIC) ? ((j + yplus) % extent[1]) : (j + yplus);
3298                 PetscInt xcoord      = (periodic && periodic[0] == DM_BOUNDARY_PERIODIC) ? ((i + xplus) % extent[0]) : (i + xplus);
3299                 PetscInt vert        = ((zcoord * extentPlus[1] + ycoord) * extentPlus[0] + xcoord) * 56 + patternvert;
3300 
3301                 cells[(k * extent[1] + j) * extent[0] + i][f][v] = vert;
3302                 seen[vert]                                       = PETSC_TRUE;
3303               }
3304             }
3305           }
3306         }
3307       }
3308       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++)
3309         if (seen[i]) numVertices++;
3310       count = 0;
3311       PetscCall(PetscMalloc1(numBlocksPlus * vertsPerBlock, &vertToTrueVert));
3312       PetscCall(PetscMalloc1(numVertices * 3, &vtxCoords));
3313       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) vertToTrueVert[i] = -1;
3314       for (PetscInt k = 0; k < extentPlus[2]; k++) {
3315         for (PetscInt j = 0; j < extentPlus[1]; j++) {
3316           for (PetscInt i = 0; i < extentPlus[0]; i++) {
3317             for (PetscInt v = 0; v < vertsPerBlock; v++) {
3318               PetscInt vIdx = ((k * extentPlus[1] + j) * extentPlus[0] + i) * vertsPerBlock + v;
3319 
3320               if (seen[vIdx]) {
3321                 PetscInt thisVert;
3322 
3323                 vertToTrueVert[vIdx] = thisVert = count++;
3324 
3325                 for (PetscInt d = 0; d < 3; d++) vtxCoords[3 * thisVert + d] = patternCoords[v][d];
3326                 vtxCoords[3 * thisVert + 0] += i * 2;
3327                 vtxCoords[3 * thisVert + 1] += j * 2;
3328                 vtxCoords[3 * thisVert + 2] += k * 2;
3329               }
3330             }
3331           }
3332         }
3333       }
3334       for (PetscInt i = 0; i < numBlocks; i++) {
3335         for (PetscInt f = 0; f < facesPerBlock; f++) {
3336           for (PetscInt v = 0; v < 4; v++) cells[i][f][v] = vertToTrueVert[cells[i][f][v]];
3337         }
3338       }
3339       PetscCall(PetscFree(vertToTrueVert));
3340       PetscCall(PetscFree(seen));
3341       cells_flat = cells[0][0];
3342       numEdges   = 0;
3343       for (PetscInt i = 0; i < numFaces; i++) {
3344         for (PetscInt e = 0; e < 4; e++) {
3345           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3346           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3347 
3348           for (PetscInt d = 0; d < 3; d++) {
3349             if (!periodic || periodic[0] != DM_BOUNDARY_PERIODIC) {
3350               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) numEdges++;
3351               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) numEdges++;
3352             }
3353           }
3354         }
3355       }
3356       PetscCall(PetscMalloc1(numEdges, &edges));
3357       PetscCall(PetscMalloc1(numEdges, &edgeSets));
3358       for (PetscInt edge = 0, i = 0; i < numFaces; i++) {
3359         for (PetscInt e = 0; e < 4; e++) {
3360           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3361           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3362 
3363           for (PetscInt d = 0; d < 3; d++) {
3364             if (!periodic || periodic[d] != DM_BOUNDARY_PERIODIC) {
3365               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) {
3366                 edges[edge][0]   = ev[0];
3367                 edges[edge][1]   = ev[1];
3368                 edgeSets[edge++] = 2 * d;
3369               }
3370               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) {
3371                 edges[edge][0]   = ev[0];
3372                 edges[edge][1]   = ev[1];
3373                 edgeSets[edge++] = 2 * d + 1;
3374               }
3375             }
3376           }
3377         }
3378       }
3379     }
3380     evalFunc   = TPSEvaluate_Gyroid;
3381     normalFunc = TPSExtrudeNormalFunc_Gyroid;
3382     break;
3383   }
3384 
3385   PetscCall(DMSetDimension(dm, topoDim));
3386   if (rank == 0) PetscCall(DMPlexBuildFromCellList(dm, numFaces, numVertices, 4, cells_flat));
3387   else PetscCall(DMPlexBuildFromCellList(dm, 0, 0, 0, NULL));
3388   PetscCall(PetscFree(cells_flat));
3389   {
3390     DM idm;
3391     PetscCall(DMPlexInterpolate(dm, &idm));
3392     PetscCall(DMPlexReplace_Internal(dm, &idm));
3393   }
3394   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, vtxCoords));
3395   else PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, NULL));
3396   PetscCall(PetscFree(vtxCoords));
3397 
3398   PetscCall(DMCreateLabel(dm, "Face Sets"));
3399   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3400   for (PetscInt e = 0; e < numEdges; e++) {
3401     PetscInt        njoin;
3402     const PetscInt *join, verts[] = {numFaces + edges[e][0], numFaces + edges[e][1]};
3403     PetscCall(DMPlexGetJoin(dm, 2, verts, &njoin, &join));
3404     PetscCheck(njoin == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected unique join of vertices %" PetscInt_FMT " and %" PetscInt_FMT, edges[e][0], edges[e][1]);
3405     PetscCall(DMLabelSetValue(label, join[0], edgeSets[e]));
3406     PetscCall(DMPlexRestoreJoin(dm, 2, verts, &njoin, &join));
3407   }
3408   PetscCall(PetscFree(edges));
3409   PetscCall(PetscFree(edgeSets));
3410   if (tps_distribute) {
3411     DM               pdm = NULL;
3412     PetscPartitioner part;
3413 
3414     PetscCall(DMPlexGetPartitioner(dm, &part));
3415     PetscCall(PetscPartitionerSetFromOptions(part));
3416     PetscCall(DMPlexDistribute(dm, 0, NULL, &pdm));
3417     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
3418     // Do not auto-distribute again
3419     PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
3420   }
3421 
3422   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3423   for (PetscInt refine = 0; refine < refinements; refine++) {
3424     PetscInt     m;
3425     DM           dmf;
3426     Vec          X;
3427     PetscScalar *x;
3428     PetscCall(DMRefine(dm, MPI_COMM_NULL, &dmf));
3429     PetscCall(DMPlexReplace_Internal(dm, &dmf));
3430 
3431     PetscCall(DMGetCoordinatesLocal(dm, &X));
3432     PetscCall(VecGetLocalSize(X, &m));
3433     PetscCall(VecGetArray(X, &x));
3434     for (PetscInt i = 0; i < m; i += 3) PetscCall(TPSNearestPoint(evalFunc, &x[i]));
3435     PetscCall(VecRestoreArray(X, &x));
3436   }
3437 
3438   // Face Sets has already been propagated to new vertices during refinement; this propagates to the initial vertices.
3439   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3440   PetscCall(DMPlexLabelComplete(dm, label));
3441 
3442   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
3443 
3444   if (thickness > 0) {
3445     DM              edm, cdm, ecdm;
3446     DMPlexTransform tr;
3447     const char     *prefix;
3448     PetscOptions    options;
3449     // Code from DMPlexExtrude
3450     PetscCall(DMPlexTransformCreate(PetscObjectComm((PetscObject)dm), &tr));
3451     PetscCall(DMPlexTransformSetDM(tr, dm));
3452     PetscCall(DMPlexTransformSetType(tr, DMPLEXEXTRUDE));
3453     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
3454     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)tr, prefix));
3455     PetscCall(PetscObjectGetOptions((PetscObject)dm, &options));
3456     PetscCall(PetscObjectSetOptions((PetscObject)tr, options));
3457     PetscCall(DMPlexTransformExtrudeSetLayers(tr, layers));
3458     PetscCall(DMPlexTransformExtrudeSetThickness(tr, thickness));
3459     PetscCall(DMPlexTransformExtrudeSetTensor(tr, PETSC_FALSE));
3460     PetscCall(DMPlexTransformExtrudeSetSymmetric(tr, PETSC_TRUE));
3461     PetscCall(DMPlexTransformExtrudeSetNormalFunction(tr, normalFunc));
3462     PetscCall(DMPlexTransformSetFromOptions(tr));
3463     PetscCall(PetscObjectSetOptions((PetscObject)tr, NULL));
3464     PetscCall(DMPlexTransformSetUp(tr));
3465     PetscCall(PetscObjectViewFromOptions((PetscObject)tr, NULL, "-dm_plex_tps_transform_view"));
3466     PetscCall(DMPlexTransformApply(tr, dm, &edm));
3467     PetscCall(DMCopyDisc(dm, edm));
3468     PetscCall(DMGetCoordinateDM(dm, &cdm));
3469     PetscCall(DMGetCoordinateDM(edm, &ecdm));
3470     PetscCall(DMCopyDisc(cdm, ecdm));
3471     PetscCall(DMPlexTransformCreateDiscLabels(tr, edm));
3472     PetscCall(DMPlexTransformDestroy(&tr));
3473     if (edm) {
3474       ((DM_Plex *)edm->data)->printFEM    = ((DM_Plex *)dm->data)->printFEM;
3475       ((DM_Plex *)edm->data)->printL2     = ((DM_Plex *)dm->data)->printL2;
3476       ((DM_Plex *)edm->data)->printLocate = ((DM_Plex *)dm->data)->printLocate;
3477     }
3478     PetscCall(DMPlexReplace_Internal(dm, &edm));
3479   }
3480   PetscFunctionReturn(PETSC_SUCCESS);
3481 }
3482 
3483 /*@
3484   DMPlexCreateTPSMesh - Create a distributed, interpolated mesh of a triply-periodic surface
3485 
3486   Collective
3487 
3488   Input Parameters:
3489 + comm           - The communicator for the `DM` object
3490 . tpstype        - Type of triply-periodic surface
3491 . extent         - Array of length 3 containing number of periods in each direction
3492 . periodic       - array of length 3 with periodicity, or `NULL` for non-periodic
3493 . tps_distribute - Distribute 2D manifold mesh prior to refinement and extrusion (more scalable)
3494 . refinements    - Number of factor-of-2 refinements of 2D manifold mesh
3495 . layers         - Number of cell layers extruded in normal direction
3496 - thickness      - Thickness in normal direction
3497 
3498   Output Parameter:
3499 . dm - The `DM` object
3500 
3501   Level: beginner
3502 
3503   Notes:
3504   This meshes the surface of the Schwarz P or Gyroid surfaces.  Schwarz P is is the simplest member of the triply-periodic minimal surfaces.
3505   https://en.wikipedia.org/wiki/Schwarz_minimal_surface#Schwarz_P_(%22Primitive%22) and can be cut with "clean" boundaries.
3506   The Gyroid (https://en.wikipedia.org/wiki/Gyroid) is another triply-periodic minimal surface with applications in additive manufacturing; it is much more difficult to "cut" since there are no planes of symmetry.
3507   Our implementation creates a very coarse mesh of the surface and refines (by 4-way splitting) as many times as requested.
3508   On each refinement, all vertices are projected to their nearest point on the surface.
3509   This projection could readily be extended to related surfaces.
3510 
3511   The face (edge) sets for the Schwarz P surface are numbered 1(-x), 2(+x), 3(-y), 4(+y), 5(-z), 6(+z).
3512   When the mesh is refined, "Face Sets" contain the new vertices (created during refinement).  Use `DMPlexLabelComplete()` to propagate to coarse-level vertices.
3513 
3514   Developer Notes:
3515   The Gyroid mesh does not currently mark boundary sets.
3516 
3517   References:
3518 . * - Maskery et al, Insights into the mechanical properties of several triply periodic minimal surface lattice structures made by polymer additive manufacturing, 2017.
3519   https://doi.org/10.1016/j.polymer.2017.11.049
3520 
3521 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMSetType()`, `DMCreate()`
3522 @*/
3523 PetscErrorCode DMPlexCreateTPSMesh(MPI_Comm comm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness, DM *dm)
3524 {
3525   PetscFunctionBegin;
3526   PetscCall(DMCreate(comm, dm));
3527   PetscCall(DMSetType(*dm, DMPLEX));
3528   PetscCall(DMPlexCreateTPSMesh_Internal(*dm, tpstype, extent, periodic, tps_distribute, refinements, layers, thickness));
3529   PetscFunctionReturn(PETSC_SUCCESS);
3530 }
3531 
3532 /*@
3533   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
3534 
3535   Collective
3536 
3537   Input Parameters:
3538 + comm    - The communicator for the `DM` object
3539 . dim     - The dimension
3540 . simplex - Use simplices, or tensor product cells
3541 - R       - The radius
3542 
3543   Output Parameter:
3544 . dm - The `DM` object
3545 
3546   Level: beginner
3547 
3548 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBallMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3549 @*/
3550 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscReal R, DM *dm)
3551 {
3552   PetscFunctionBegin;
3553   PetscAssertPointer(dm, 5);
3554   PetscCall(DMCreate(comm, dm));
3555   PetscCall(DMSetType(*dm, DMPLEX));
3556   PetscCall(DMPlexCreateSphereMesh_Internal(*dm, dim, simplex, R));
3557   PetscFunctionReturn(PETSC_SUCCESS);
3558 }
3559 
3560 static PetscErrorCode DMPlexCreateBallMesh_Internal(DM dm, PetscInt dim, PetscReal R)
3561 {
3562   DM      sdm, vol;
3563   DMLabel bdlabel;
3564 
3565   PetscFunctionBegin;
3566   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
3567   PetscCall(DMSetType(sdm, DMPLEX));
3568   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sdm, "bd_"));
3569   PetscCall(DMPlexCreateSphereMesh_Internal(sdm, dim - 1, PETSC_TRUE, R));
3570   PetscCall(DMSetFromOptions(sdm));
3571   PetscCall(DMViewFromOptions(sdm, NULL, "-dm_view"));
3572   PetscCall(DMPlexGenerate(sdm, NULL, PETSC_TRUE, &vol));
3573   PetscCall(DMDestroy(&sdm));
3574   PetscCall(DMPlexReplace_Internal(dm, &vol));
3575   PetscCall(DMCreateLabel(dm, "marker"));
3576   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
3577   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
3578   PetscCall(DMPlexLabelComplete(dm, bdlabel));
3579   PetscFunctionReturn(PETSC_SUCCESS);
3580 }
3581 
3582 /*@
3583   DMPlexCreateBallMesh - Creates a simplex mesh on the d-dimensional ball, B^d.
3584 
3585   Collective
3586 
3587   Input Parameters:
3588 + comm - The communicator for the `DM` object
3589 . dim  - The dimension
3590 - R    - The radius
3591 
3592   Output Parameter:
3593 . dm - The `DM` object
3594 
3595   Options Database Key:
3596 . bd_dm_refine - This will refine the surface mesh preserving the sphere geometry
3597 
3598   Level: beginner
3599 
3600 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3601 @*/
3602 PetscErrorCode DMPlexCreateBallMesh(MPI_Comm comm, PetscInt dim, PetscReal R, DM *dm)
3603 {
3604   PetscFunctionBegin;
3605   PetscCall(DMCreate(comm, dm));
3606   PetscCall(DMSetType(*dm, DMPLEX));
3607   PetscCall(DMPlexCreateBallMesh_Internal(*dm, dim, R));
3608   PetscFunctionReturn(PETSC_SUCCESS);
3609 }
3610 
3611 static PetscErrorCode DMPlexCreateReferenceCell_Internal(DM rdm, DMPolytopeType ct)
3612 {
3613   PetscFunctionBegin;
3614   switch (ct) {
3615   case DM_POLYTOPE_POINT: {
3616     PetscInt    numPoints[1]        = {1};
3617     PetscInt    coneSize[1]         = {0};
3618     PetscInt    cones[1]            = {0};
3619     PetscInt    coneOrientations[1] = {0};
3620     PetscScalar vertexCoords[1]     = {0.0};
3621 
3622     PetscCall(DMSetDimension(rdm, 0));
3623     PetscCall(DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3624   } break;
3625   case DM_POLYTOPE_SEGMENT: {
3626     PetscInt    numPoints[2]        = {2, 1};
3627     PetscInt    coneSize[3]         = {2, 0, 0};
3628     PetscInt    cones[2]            = {1, 2};
3629     PetscInt    coneOrientations[2] = {0, 0};
3630     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3631 
3632     PetscCall(DMSetDimension(rdm, 1));
3633     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3634   } break;
3635   case DM_POLYTOPE_POINT_PRISM_TENSOR: {
3636     PetscInt    numPoints[2]        = {2, 1};
3637     PetscInt    coneSize[3]         = {2, 0, 0};
3638     PetscInt    cones[2]            = {1, 2};
3639     PetscInt    coneOrientations[2] = {0, 0};
3640     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3641 
3642     PetscCall(DMSetDimension(rdm, 1));
3643     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3644   } break;
3645   case DM_POLYTOPE_TRIANGLE: {
3646     PetscInt    numPoints[2]        = {3, 1};
3647     PetscInt    coneSize[4]         = {3, 0, 0, 0};
3648     PetscInt    cones[3]            = {1, 2, 3};
3649     PetscInt    coneOrientations[3] = {0, 0, 0};
3650     PetscScalar vertexCoords[6]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0};
3651 
3652     PetscCall(DMSetDimension(rdm, 2));
3653     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3654   } break;
3655   case DM_POLYTOPE_QUADRILATERAL: {
3656     PetscInt    numPoints[2]        = {4, 1};
3657     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3658     PetscInt    cones[4]            = {1, 2, 3, 4};
3659     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3660     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0};
3661 
3662     PetscCall(DMSetDimension(rdm, 2));
3663     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3664   } break;
3665   case DM_POLYTOPE_SEG_PRISM_TENSOR: {
3666     PetscInt    numPoints[2]        = {4, 1};
3667     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3668     PetscInt    cones[4]            = {1, 2, 3, 4};
3669     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3670     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0};
3671 
3672     PetscCall(DMSetDimension(rdm, 2));
3673     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3674   } break;
3675   case DM_POLYTOPE_TETRAHEDRON: {
3676     PetscInt    numPoints[2]        = {4, 1};
3677     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3678     PetscInt    cones[4]            = {1, 2, 3, 4};
3679     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3680     PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0};
3681 
3682     PetscCall(DMSetDimension(rdm, 3));
3683     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3684   } break;
3685   case DM_POLYTOPE_HEXAHEDRON: {
3686     PetscInt    numPoints[2]        = {8, 1};
3687     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3688     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3689     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3690     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3691 
3692     PetscCall(DMSetDimension(rdm, 3));
3693     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3694   } break;
3695   case DM_POLYTOPE_TRI_PRISM: {
3696     PetscInt    numPoints[2]        = {6, 1};
3697     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3698     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3699     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3700     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3701 
3702     PetscCall(DMSetDimension(rdm, 3));
3703     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3704   } break;
3705   case DM_POLYTOPE_TRI_PRISM_TENSOR: {
3706     PetscInt    numPoints[2]        = {6, 1};
3707     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3708     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3709     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3710     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3711 
3712     PetscCall(DMSetDimension(rdm, 3));
3713     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3714   } break;
3715   case DM_POLYTOPE_QUAD_PRISM_TENSOR: {
3716     PetscInt    numPoints[2]        = {8, 1};
3717     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3718     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3719     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3720     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3721 
3722     PetscCall(DMSetDimension(rdm, 3));
3723     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3724   } break;
3725   case DM_POLYTOPE_PYRAMID: {
3726     PetscInt    numPoints[2]        = {5, 1};
3727     PetscInt    coneSize[6]         = {5, 0, 0, 0, 0, 0};
3728     PetscInt    cones[5]            = {1, 2, 3, 4, 5};
3729     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3730     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, 0.0, 0.0, 1.0};
3731 
3732     PetscCall(DMSetDimension(rdm, 3));
3733     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3734   } break;
3735   default:
3736     SETERRQ(PetscObjectComm((PetscObject)rdm), PETSC_ERR_ARG_WRONG, "Cannot create reference cell for cell type %s", DMPolytopeTypes[ct]);
3737   }
3738   {
3739     PetscInt Nv, v;
3740 
3741     /* Must create the celltype label here so that we do not automatically try to compute the types */
3742     PetscCall(DMCreateLabel(rdm, "celltype"));
3743     PetscCall(DMPlexSetCellType(rdm, 0, ct));
3744     PetscCall(DMPlexGetChart(rdm, NULL, &Nv));
3745     for (v = 1; v < Nv; ++v) PetscCall(DMPlexSetCellType(rdm, v, DM_POLYTOPE_POINT));
3746   }
3747   PetscCall(DMPlexInterpolateInPlace_Internal(rdm));
3748   PetscCall(PetscObjectSetName((PetscObject)rdm, DMPolytopeTypes[ct]));
3749   PetscFunctionReturn(PETSC_SUCCESS);
3750 }
3751 
3752 /*@
3753   DMPlexCreateReferenceCell - Create a `DMPLEX` with the appropriate FEM reference cell
3754 
3755   Collective
3756 
3757   Input Parameters:
3758 + comm - The communicator
3759 - ct   - The cell type of the reference cell
3760 
3761   Output Parameter:
3762 . refdm - The reference cell
3763 
3764   Level: intermediate
3765 
3766 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBoxMesh()`
3767 @*/
3768 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, DMPolytopeType ct, DM *refdm)
3769 {
3770   PetscFunctionBegin;
3771   PetscCall(DMCreate(comm, refdm));
3772   PetscCall(DMSetType(*refdm, DMPLEX));
3773   PetscCall(DMPlexCreateReferenceCell_Internal(*refdm, ct));
3774   PetscFunctionReturn(PETSC_SUCCESS);
3775 }
3776 
3777 static PetscErrorCode DMPlexCreateBoundaryLabel_Private(DM dm, const char name[])
3778 {
3779   DM        plex;
3780   DMLabel   label;
3781   PetscBool hasLabel;
3782 
3783   PetscFunctionBegin;
3784   PetscCall(DMHasLabel(dm, name, &hasLabel));
3785   if (hasLabel) PetscFunctionReturn(PETSC_SUCCESS);
3786   PetscCall(DMCreateLabel(dm, name));
3787   PetscCall(DMGetLabel(dm, name, &label));
3788   PetscCall(DMConvert(dm, DMPLEX, &plex));
3789   PetscCall(DMPlexMarkBoundaryFaces(plex, 1, label));
3790   PetscCall(DMPlexLabelComplete(plex, label));
3791   PetscCall(DMDestroy(&plex));
3792   PetscFunctionReturn(PETSC_SUCCESS);
3793 }
3794 
3795 /*
3796   We use the last coordinate as the radius, the inner radius is lower[dim-1] and the outer radius is upper[dim-1]. Then we map the first coordinate around the circle.
3797 
3798     (x, y) -> (r, theta) = (x[1], (x[0] - lower[0]) * 2\pi/(upper[0] - lower[0]))
3799 */
3800 static void boxToAnnulus(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
3801 {
3802   const PetscReal low = PetscRealPart(constants[0]);
3803   const PetscReal upp = PetscRealPart(constants[1]);
3804   const PetscReal r   = PetscRealPart(u[1]);
3805   const PetscReal th  = 2. * PETSC_PI * (PetscRealPart(u[0]) - low) / (upp - low);
3806 
3807   f0[0] = r * PetscCosReal(th);
3808   f0[1] = r * PetscSinReal(th);
3809 }
3810 
3811 const char *const DMPlexShapes[] = {"box", "box_surface", "ball", "sphere", "cylinder", "schwarz_p", "gyroid", "doublet", "annulus", "hypercubic", "zbox", "unknown", "DMPlexShape", "DM_SHAPE_", NULL};
3812 
3813 static PetscErrorCode DMPlexCreateFromOptions_Internal(PetscOptionItems *PetscOptionsObject, PetscBool *useCoordSpace, DM dm)
3814 {
3815   DMPlexShape    shape   = DM_SHAPE_BOX;
3816   DMPolytopeType cell    = DM_POLYTOPE_TRIANGLE;
3817   PetscInt       dim     = 2;
3818   PetscBool      simplex = PETSC_TRUE, interpolate = PETSC_TRUE, adjCone = PETSC_FALSE, adjClosure = PETSC_TRUE, refDomain = PETSC_FALSE;
3819   PetscBool      flg, flg2, fflg, bdfflg, nameflg;
3820   MPI_Comm       comm;
3821   char           filename[PETSC_MAX_PATH_LEN]   = "<unspecified>";
3822   char           bdFilename[PETSC_MAX_PATH_LEN] = "<unspecified>";
3823   char           plexname[PETSC_MAX_PATH_LEN]   = "";
3824 
3825   PetscFunctionBegin;
3826   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromOptions, dm, 0, 0, 0));
3827   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3828   /* TODO Turn this into a registration interface */
3829   PetscCall(PetscOptionsString("-dm_plex_filename", "File containing a mesh", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &fflg));
3830   PetscCall(PetscOptionsString("-dm_plex_boundary_filename", "File containing a mesh boundary", "DMPlexCreateFromFile", bdFilename, bdFilename, sizeof(bdFilename), &bdfflg));
3831   PetscCall(PetscOptionsString("-dm_plex_name", "Name of the mesh in the file", "DMPlexCreateFromFile", plexname, plexname, sizeof(plexname), &nameflg));
3832   PetscCall(PetscOptionsEnum("-dm_plex_cell", "Cell shape", "", DMPolytopeTypes, (PetscEnum)cell, (PetscEnum *)&cell, NULL));
3833   PetscCall(PetscOptionsBool("-dm_plex_reference_cell_domain", "Use a reference cell domain", "", refDomain, &refDomain, NULL));
3834   PetscCall(PetscOptionsEnum("-dm_plex_shape", "Shape for built-in mesh", "", DMPlexShapes, (PetscEnum)shape, (PetscEnum *)&shape, &flg));
3835   PetscCall(PetscOptionsBoundedInt("-dm_plex_dim", "Topological dimension of the mesh", "DMGetDimension", dim, &dim, &flg, 0));
3836   PetscCheck(dim >= 0, comm, PETSC_ERR_ARG_OUTOFRANGE, "Dimension %" PetscInt_FMT " should be in [0, infinity)", dim);
3837   PetscCall(PetscOptionsBool("-dm_plex_simplex", "Mesh cell shape", "", simplex, &simplex, &flg));
3838   PetscCall(PetscOptionsBool("-dm_plex_interpolate", "Flag to create edges and faces automatically", "", interpolate, &interpolate, &flg));
3839   PetscCall(PetscOptionsBool("-dm_plex_adj_cone", "Set adjacency direction", "DMSetBasicAdjacency", adjCone, &adjCone, &flg));
3840   PetscCall(PetscOptionsBool("-dm_plex_adj_closure", "Set adjacency size", "DMSetBasicAdjacency", adjClosure, &adjClosure, &flg2));
3841   if (flg || flg2) PetscCall(DMSetBasicAdjacency(dm, adjCone, adjClosure));
3842 
3843   switch (cell) {
3844   case DM_POLYTOPE_POINT:
3845   case DM_POLYTOPE_SEGMENT:
3846   case DM_POLYTOPE_POINT_PRISM_TENSOR:
3847   case DM_POLYTOPE_TRIANGLE:
3848   case DM_POLYTOPE_QUADRILATERAL:
3849   case DM_POLYTOPE_TETRAHEDRON:
3850   case DM_POLYTOPE_HEXAHEDRON:
3851     *useCoordSpace = PETSC_TRUE;
3852     break;
3853   default:
3854     *useCoordSpace = PETSC_FALSE;
3855     break;
3856   }
3857 
3858   if (fflg) {
3859     DM dmnew;
3860 
3861     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), filename, plexname, interpolate, &dmnew));
3862     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3863     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3864   } else if (refDomain) {
3865     PetscCall(DMPlexCreateReferenceCell_Internal(dm, cell));
3866   } else if (bdfflg) {
3867     DM bdm, dmnew;
3868 
3869     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), bdFilename, plexname, interpolate, &bdm));
3870     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)bdm, "bd_"));
3871     PetscCall(DMSetFromOptions(bdm));
3872     PetscCall(DMPlexGenerate(bdm, NULL, interpolate, &dmnew));
3873     PetscCall(DMDestroy(&bdm));
3874     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3875     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3876   } else {
3877     PetscCall(PetscObjectSetName((PetscObject)dm, DMPlexShapes[shape]));
3878     switch (shape) {
3879     case DM_SHAPE_BOX:
3880     case DM_SHAPE_ZBOX:
3881     case DM_SHAPE_ANNULUS: {
3882       PetscInt       faces[3]  = {0, 0, 0};
3883       PetscReal      lower[3]  = {0, 0, 0};
3884       PetscReal      upper[3]  = {1, 1, 1};
3885       DMBoundaryType bdt[3]    = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3886       PetscBool      isAnnular = shape == DM_SHAPE_ANNULUS ? PETSC_TRUE : PETSC_FALSE;
3887       PetscInt       i, n;
3888 
3889       n = dim;
3890       for (i = 0; i < dim; ++i) faces[i] = (dim == 1 ? 1 : 4 - dim);
3891       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3892       n = 3;
3893       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3894       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3895       n = 3;
3896       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3897       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3898       n = 3;
3899       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
3900       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3901 
3902       PetscCheck(!isAnnular || dim == 2, comm, PETSC_ERR_ARG_OUTOFRANGE, "Only two dimensional annuli have been implemented");
3903       if (isAnnular)
3904         for (i = 0; i < dim - 1; ++i) bdt[i] = DM_BOUNDARY_PERIODIC;
3905 
3906       switch (cell) {
3907       case DM_POLYTOPE_TRI_PRISM_TENSOR:
3908         PetscCall(DMPlexCreateWedgeBoxMesh_Internal(dm, faces, lower, upper, bdt));
3909         if (!interpolate) {
3910           DM udm;
3911 
3912           PetscCall(DMPlexUninterpolate(dm, &udm));
3913           PetscCall(DMPlexReplace_Internal(dm, &udm));
3914         }
3915         break;
3916       default:
3917         PetscCall(DMPlexCreateBoxMesh_Internal(dm, shape, dim, simplex, faces, lower, upper, bdt, interpolate));
3918         break;
3919       }
3920       if (isAnnular) {
3921         DM          cdm;
3922         PetscDS     cds;
3923         PetscScalar bounds[2] = {lower[0], upper[0]};
3924 
3925         // Fix coordinates for annular region
3926         PetscCall(DMSetPeriodicity(dm, NULL, NULL, NULL));
3927         PetscCall(DMSetCellCoordinatesLocal(dm, NULL));
3928         PetscCall(DMSetCellCoordinates(dm, NULL));
3929         PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_TRUE, NULL));
3930         PetscCall(DMGetCoordinateDM(dm, &cdm));
3931         PetscCall(DMGetDS(cdm, &cds));
3932         PetscCall(PetscDSSetConstants(cds, 2, bounds));
3933         PetscCall(DMPlexRemapGeometry(dm, 0.0, boxToAnnulus));
3934       }
3935     } break;
3936     case DM_SHAPE_BOX_SURFACE: {
3937       PetscInt  faces[3] = {0, 0, 0};
3938       PetscReal lower[3] = {0, 0, 0};
3939       PetscReal upper[3] = {1, 1, 1};
3940       PetscInt  i, n;
3941 
3942       n = dim + 1;
3943       for (i = 0; i < dim + 1; ++i) faces[i] = (dim + 1 == 1 ? 1 : 4 - (dim + 1));
3944       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3945       n = 3;
3946       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3947       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3948       n = 3;
3949       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3950       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3951       PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(dm, dim + 1, faces, lower, upper, interpolate));
3952     } break;
3953     case DM_SHAPE_SPHERE: {
3954       PetscReal R = 1.0;
3955 
3956       PetscCall(PetscOptionsReal("-dm_plex_sphere_radius", "Radius of the sphere", "", R, &R, &flg));
3957       PetscCall(DMPlexCreateSphereMesh_Internal(dm, dim, simplex, R));
3958     } break;
3959     case DM_SHAPE_BALL: {
3960       PetscReal R = 1.0;
3961 
3962       PetscCall(PetscOptionsReal("-dm_plex_ball_radius", "Radius of the ball", "", R, &R, &flg));
3963       PetscCall(DMPlexCreateBallMesh_Internal(dm, dim, R));
3964     } break;
3965     case DM_SHAPE_CYLINDER: {
3966       DMBoundaryType bdt = DM_BOUNDARY_NONE;
3967       PetscInt       Nw  = 6;
3968 
3969       PetscCall(PetscOptionsEnum("-dm_plex_cylinder_bd", "Boundary type in the z direction", "", DMBoundaryTypes, (PetscEnum)bdt, (PetscEnum *)&bdt, NULL));
3970       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_wedges", "Number of wedges around the cylinder", "", Nw, &Nw, NULL));
3971       switch (cell) {
3972       case DM_POLYTOPE_TRI_PRISM_TENSOR:
3973         PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(dm, Nw, interpolate));
3974         break;
3975       default:
3976         PetscCall(DMPlexCreateHexCylinderMesh_Internal(dm, bdt));
3977         break;
3978       }
3979     } break;
3980     case DM_SHAPE_SCHWARZ_P: // fallthrough
3981     case DM_SHAPE_GYROID: {
3982       PetscInt       extent[3] = {1, 1, 1}, refine = 0, layers = 0, three;
3983       PetscReal      thickness   = 0.;
3984       DMBoundaryType periodic[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3985       DMPlexTPSType  tps_type    = shape == DM_SHAPE_SCHWARZ_P ? DMPLEX_TPS_SCHWARZ_P : DMPLEX_TPS_GYROID;
3986       PetscBool      tps_distribute;
3987       PetscCall(PetscOptionsIntArray("-dm_plex_tps_extent", "Number of replicas for each of three dimensions", NULL, extent, (three = 3, &three), NULL));
3988       PetscCall(PetscOptionsInt("-dm_plex_tps_refine", "Number of refinements", NULL, refine, &refine, NULL));
3989       PetscCall(PetscOptionsEnumArray("-dm_plex_tps_periodic", "Periodicity in each of three dimensions", NULL, DMBoundaryTypes, (PetscEnum *)periodic, (three = 3, &three), NULL));
3990       PetscCall(PetscOptionsInt("-dm_plex_tps_layers", "Number of layers in volumetric extrusion (or zero to not extrude)", NULL, layers, &layers, NULL));
3991       PetscCall(PetscOptionsReal("-dm_plex_tps_thickness", "Thickness of volumetric extrusion", NULL, thickness, &thickness, NULL));
3992       PetscCall(DMPlexDistributeGetDefault(dm, &tps_distribute));
3993       PetscCall(PetscOptionsBool("-dm_plex_tps_distribute", "Distribute the 2D mesh prior to refinement and extrusion", NULL, tps_distribute, &tps_distribute, NULL));
3994       PetscCall(DMPlexCreateTPSMesh_Internal(dm, tps_type, extent, periodic, tps_distribute, refine, layers, thickness));
3995     } break;
3996     case DM_SHAPE_DOUBLET: {
3997       DM        dmnew;
3998       PetscReal rl = 0.0;
3999 
4000       PetscCall(PetscOptionsReal("-dm_plex_doublet_refinementlimit", "Refinement limit", NULL, rl, &rl, NULL));
4001       PetscCall(DMPlexCreateDoublet(PetscObjectComm((PetscObject)dm), dim, simplex, interpolate, rl, &dmnew));
4002       PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4003       PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4004     } break;
4005     case DM_SHAPE_HYPERCUBIC: {
4006       PetscInt       *edges;
4007       PetscReal      *lower, *upper;
4008       DMBoundaryType *bdt;
4009       PetscInt        n, d;
4010 
4011       *useCoordSpace = PETSC_FALSE;
4012       PetscCall(PetscMalloc4(dim, &edges, dim, &lower, dim, &upper, dim, &bdt));
4013       for (d = 0; d < dim; ++d) {
4014         edges[d] = 1;
4015         lower[d] = 0.;
4016         upper[d] = 1.;
4017         bdt[d]   = DM_BOUNDARY_PERIODIC;
4018       }
4019       n = dim;
4020       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", edges, &n, &flg));
4021       n = dim;
4022       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
4023       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4024       n = dim;
4025       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
4026       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4027       n = dim;
4028       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
4029       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4030       PetscCall(DMPlexCreateHypercubicMesh_Internal(dm, dim, lower, upper, edges, bdt));
4031       PetscCall(PetscFree4(edges, lower, upper, bdt));
4032     } break;
4033     default:
4034       SETERRQ(comm, PETSC_ERR_SUP, "Domain shape %s is unsupported", DMPlexShapes[shape]);
4035     }
4036   }
4037   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
4038   if (!((PetscObject)dm)->name && nameflg) PetscCall(PetscObjectSetName((PetscObject)dm, plexname));
4039   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromOptions, dm, 0, 0, 0));
4040   PetscFunctionReturn(PETSC_SUCCESS);
4041 }
4042 
4043 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
4044 {
4045   DM_Plex  *mesh = (DM_Plex *)dm->data;
4046   PetscBool flg, flg2;
4047   char      bdLabel[PETSC_MAX_PATH_LEN];
4048 
4049   PetscFunctionBegin;
4050   /* Handle viewing */
4051   PetscCall(PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL));
4052   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fem", "Debug output level all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL, 0));
4053   PetscCall(PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL));
4054   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL, 0));
4055   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_locate", "Debug output level all point location computations", "DMLocatePoints", 0, &mesh->printLocate, NULL, 0));
4056   PetscCall(DMMonitorSetFromOptions(dm, "-dm_plex_monitor_throughput", "Monitor the simulation throughput", "DMPlexMonitorThroughput", DMPlexMonitorThroughput, NULL, &flg));
4057   if (flg) PetscCall(PetscLogDefaultBegin());
4058   /* Labeling */
4059   PetscCall(PetscOptionsString("-dm_plex_boundary_label", "Label to mark the mesh boundary", "", bdLabel, bdLabel, sizeof(bdLabel), &flg));
4060   if (flg) PetscCall(DMPlexCreateBoundaryLabel_Private(dm, bdLabel));
4061   /* Point Location */
4062   PetscCall(PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL));
4063   /* Partitioning and distribution */
4064   PetscCall(PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL));
4065   /* Generation and remeshing */
4066   PetscCall(PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL));
4067   /* Projection behavior */
4068   PetscCall(PetscOptionsBoundedInt("-dm_plex_max_projection_height", "Maximum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL, 0));
4069   PetscCall(PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL));
4070   /* Checking structure */
4071   {
4072     PetscBool all = PETSC_FALSE;
4073 
4074     PetscCall(PetscOptionsBool("-dm_plex_check_all", "Perform all basic checks", "DMPlexCheck", PETSC_FALSE, &all, NULL));
4075     if (all) {
4076       PetscCall(DMPlexCheck(dm));
4077     } else {
4078       PetscCall(PetscOptionsBool("-dm_plex_check_symmetry", "Check that the adjacency information in the mesh is symmetric", "DMPlexCheckSymmetry", PETSC_FALSE, &flg, &flg2));
4079       if (flg && flg2) PetscCall(DMPlexCheckSymmetry(dm));
4080       PetscCall(PetscOptionsBool("-dm_plex_check_skeleton", "Check that each cell has the correct number of vertices (only for homogeneous simplex or tensor meshes)", "DMPlexCheckSkeleton", PETSC_FALSE, &flg, &flg2));
4081       if (flg && flg2) PetscCall(DMPlexCheckSkeleton(dm, 0));
4082       PetscCall(PetscOptionsBool("-dm_plex_check_faces", "Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type", "DMPlexCheckFaces", PETSC_FALSE, &flg, &flg2));
4083       if (flg && flg2) PetscCall(DMPlexCheckFaces(dm, 0));
4084       PetscCall(PetscOptionsBool("-dm_plex_check_geometry", "Check that cells have positive volume", "DMPlexCheckGeometry", PETSC_FALSE, &flg, &flg2));
4085       if (flg && flg2) PetscCall(DMPlexCheckGeometry(dm));
4086       PetscCall(PetscOptionsBool("-dm_plex_check_pointsf", "Check some necessary conditions for PointSF", "DMPlexCheckPointSF", PETSC_FALSE, &flg, &flg2));
4087       if (flg && flg2) PetscCall(DMPlexCheckPointSF(dm, NULL, PETSC_FALSE));
4088       PetscCall(PetscOptionsBool("-dm_plex_check_interface_cones", "Check points on inter-partition interfaces have conforming order of cone points", "DMPlexCheckInterfaceCones", PETSC_FALSE, &flg, &flg2));
4089       if (flg && flg2) PetscCall(DMPlexCheckInterfaceCones(dm));
4090     }
4091     PetscCall(PetscOptionsBool("-dm_plex_check_cell_shape", "Check cell shape", "DMPlexCheckCellShape", PETSC_FALSE, &flg, &flg2));
4092     if (flg && flg2) PetscCall(DMPlexCheckCellShape(dm, PETSC_TRUE, PETSC_DETERMINE));
4093   }
4094   {
4095     PetscReal scale = 1.0;
4096 
4097     PetscCall(PetscOptionsReal("-dm_plex_scale", "Scale factor for mesh coordinates", "DMPlexScale", scale, &scale, &flg));
4098     if (flg) {
4099       Vec coordinates, coordinatesLocal;
4100 
4101       PetscCall(DMGetCoordinates(dm, &coordinates));
4102       PetscCall(DMGetCoordinatesLocal(dm, &coordinatesLocal));
4103       PetscCall(VecScale(coordinates, scale));
4104       PetscCall(VecScale(coordinatesLocal, scale));
4105     }
4106   }
4107   PetscCall(PetscPartitionerSetFromOptions(mesh->partitioner));
4108   PetscFunctionReturn(PETSC_SUCCESS);
4109 }
4110 
4111 PetscErrorCode DMSetFromOptions_Overlap_Plex(DM dm, PetscOptionItems *PetscOptionsObject, PetscInt *overlap)
4112 {
4113   PetscInt  numOvLabels = 16, numOvExLabels = 16;
4114   char     *ovLabelNames[16], *ovExLabelNames[16];
4115   PetscInt  numOvValues = 16, numOvExValues = 16, l;
4116   PetscBool flg;
4117 
4118   PetscFunctionBegin;
4119   PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", *overlap, overlap, NULL, 0));
4120   PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_labels", "List of overlap label names", "DMPlexDistribute", ovLabelNames, &numOvLabels, &flg));
4121   if (!flg) numOvLabels = 0;
4122   if (numOvLabels) {
4123     ((DM_Plex *)dm->data)->numOvLabels = numOvLabels;
4124     for (l = 0; l < numOvLabels; ++l) {
4125       PetscCall(DMGetLabel(dm, ovLabelNames[l], &((DM_Plex *)dm->data)->ovLabels[l]));
4126       PetscCheck(((DM_Plex *)dm->data)->ovLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovLabelNames[l]);
4127       PetscCall(PetscFree(ovLabelNames[l]));
4128     }
4129     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_values", "List of overlap label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovValues, &numOvValues, &flg));
4130     if (!flg) numOvValues = 0;
4131     PetscCheck(numOvLabels == numOvValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvLabels, numOvValues);
4132 
4133     PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_exclude_labels", "List of overlap exclude label names", "DMPlexDistribute", ovExLabelNames, &numOvExLabels, &flg));
4134     if (!flg) numOvExLabels = 0;
4135     ((DM_Plex *)dm->data)->numOvExLabels = numOvExLabels;
4136     for (l = 0; l < numOvExLabels; ++l) {
4137       PetscCall(DMGetLabel(dm, ovExLabelNames[l], &((DM_Plex *)dm->data)->ovExLabels[l]));
4138       PetscCheck(((DM_Plex *)dm->data)->ovExLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovExLabelNames[l]);
4139       PetscCall(PetscFree(ovExLabelNames[l]));
4140     }
4141     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_exclude_values", "List of overlap exclude label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovExValues, &numOvExValues, &flg));
4142     if (!flg) numOvExValues = 0;
4143     PetscCheck(numOvExLabels == numOvExValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of exclude labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvExLabels, numOvExValues);
4144   }
4145   PetscFunctionReturn(PETSC_SUCCESS);
4146 }
4147 
4148 static PetscErrorCode DMSetFromOptions_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
4149 {
4150   PetscFunctionList        ordlist;
4151   char                     oname[256];
4152   PetscReal                volume    = -1.0;
4153   PetscInt                 prerefine = 0, refine = 0, r, coarsen = 0, overlap = 0, extLayers = 0, dim;
4154   PetscBool                uniformOrig, created = PETSC_FALSE, uniform = PETSC_TRUE, distribute, interpolate = PETSC_TRUE, coordSpace = PETSC_TRUE, remap = PETSC_TRUE, ghostCells = PETSC_FALSE, isHierarchy, ignoreModel = PETSC_FALSE, flg;
4155   DMPlexReorderDefaultFlag reorder;
4156 
4157   PetscFunctionBegin;
4158   PetscOptionsHeadBegin(PetscOptionsObject, "DMPlex Options");
4159   if (dm->cloneOpts) goto non_refine;
4160   /* Handle automatic creation */
4161   PetscCall(DMGetDimension(dm, &dim));
4162   if (dim < 0) {
4163     PetscCall(DMPlexCreateFromOptions_Internal(PetscOptionsObject, &coordSpace, dm));
4164     created = PETSC_TRUE;
4165   }
4166   PetscCall(DMGetDimension(dm, &dim));
4167   /* Handle interpolation before distribution */
4168   PetscCall(PetscOptionsBool("-dm_plex_interpolate_pre", "Flag to interpolate mesh before distribution", "", interpolate, &interpolate, &flg));
4169   if (flg) {
4170     DMPlexInterpolatedFlag interpolated;
4171 
4172     PetscCall(DMPlexIsInterpolated(dm, &interpolated));
4173     if (interpolated == DMPLEX_INTERPOLATED_FULL && !interpolate) {
4174       DM udm;
4175 
4176       PetscCall(DMPlexUninterpolate(dm, &udm));
4177       PetscCall(DMPlexReplace_Internal(dm, &udm));
4178     } else if (interpolated != DMPLEX_INTERPOLATED_FULL && interpolate) {
4179       DM idm;
4180 
4181       PetscCall(DMPlexInterpolate(dm, &idm));
4182       PetscCall(DMPlexReplace_Internal(dm, &idm));
4183     }
4184   }
4185   /* Handle DMPlex refinement before distribution */
4186   PetscCall(PetscOptionsBool("-dm_refine_ignore_model", "Flag to ignore the geometry model when refining", "DMCreate", ignoreModel, &ignoreModel, &flg));
4187   if (flg) ((DM_Plex *)dm->data)->ignoreModel = ignoreModel;
4188   PetscCall(DMPlexGetRefinementUniform(dm, &uniformOrig));
4189   PetscCall(PetscOptionsBoundedInt("-dm_refine_pre", "The number of refinements before distribution", "DMCreate", prerefine, &prerefine, NULL, 0));
4190   PetscCall(PetscOptionsBool("-dm_refine_remap_pre", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
4191   PetscCall(PetscOptionsBool("-dm_refine_uniform_pre", "Flag for uniform refinement before distribution", "DMCreate", uniform, &uniform, &flg));
4192   if (flg) PetscCall(DMPlexSetRefinementUniform(dm, uniform));
4193   PetscCall(PetscOptionsReal("-dm_refine_volume_limit_pre", "The maximum cell volume after refinement before distribution", "DMCreate", volume, &volume, &flg));
4194   if (flg) {
4195     PetscCall(DMPlexSetRefinementUniform(dm, PETSC_FALSE));
4196     PetscCall(DMPlexSetRefinementLimit(dm, volume));
4197     prerefine = PetscMax(prerefine, 1);
4198   }
4199   for (r = 0; r < prerefine; ++r) {
4200     DM             rdm;
4201     PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4202 
4203     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4204     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
4205     PetscCall(DMPlexReplace_Internal(dm, &rdm));
4206     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4207     if (coordFunc && remap) {
4208       PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4209       ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4210     }
4211   }
4212   PetscCall(DMPlexSetRefinementUniform(dm, uniformOrig));
4213   /* Handle DMPlex extrusion before distribution */
4214   PetscCall(PetscOptionsBoundedInt("-dm_extrude", "The number of layers to extrude", "", extLayers, &extLayers, NULL, 0));
4215   if (extLayers) {
4216     DM edm;
4217 
4218     PetscCall(DMExtrude(dm, extLayers, &edm));
4219     PetscCall(DMPlexReplace_Internal(dm, &edm));
4220     ((DM_Plex *)dm->data)->coordFunc = NULL;
4221     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4222     extLayers = 0;
4223     PetscCall(DMGetDimension(dm, &dim));
4224   }
4225   /* Handle DMPlex reordering before distribution */
4226   PetscCall(DMPlexReorderGetDefault(dm, &reorder));
4227   PetscCall(MatGetOrderingList(&ordlist));
4228   PetscCall(PetscStrncpy(oname, MATORDERINGNATURAL, sizeof(oname)));
4229   PetscCall(PetscOptionsFList("-dm_plex_reorder", "Set mesh reordering type", "DMPlexGetOrdering", ordlist, MATORDERINGNATURAL, oname, sizeof(oname), &flg));
4230   if (reorder == DMPLEX_REORDER_DEFAULT_TRUE || flg) {
4231     DM pdm;
4232     IS perm;
4233 
4234     PetscCall(DMPlexGetOrdering(dm, oname, NULL, &perm));
4235     PetscCall(DMPlexPermute(dm, perm, &pdm));
4236     PetscCall(ISDestroy(&perm));
4237     PetscCall(DMPlexReplace_Internal(dm, &pdm));
4238     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4239   }
4240   /* Handle DMPlex distribution */
4241   PetscCall(DMPlexDistributeGetDefault(dm, &distribute));
4242   PetscCall(PetscOptionsBool("-dm_distribute", "Flag to redistribute a mesh among processes", "DMPlexDistribute", distribute, &distribute, NULL));
4243   PetscCall(DMSetFromOptions_Overlap_Plex(dm, PetscOptionsObject, &overlap));
4244   if (distribute) {
4245     DM               pdm = NULL;
4246     PetscPartitioner part;
4247 
4248     PetscCall(DMPlexGetPartitioner(dm, &part));
4249     PetscCall(PetscPartitionerSetFromOptions(part));
4250     PetscCall(DMPlexDistribute(dm, overlap, NULL, &pdm));
4251     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
4252   }
4253   /* Must check CEED options before creating function space for coordinates */
4254   {
4255     PetscBool useCeed = PETSC_FALSE, flg;
4256 
4257     PetscCall(PetscOptionsBool("-dm_plex_use_ceed", "Use LibCEED as the FEM backend", "DMPlexSetUseCeed", useCeed, &useCeed, &flg));
4258     if (flg) PetscCall(DMPlexSetUseCeed(dm, useCeed));
4259   }
4260   /* Create coordinate space */
4261   if (created) {
4262     DM_Plex  *mesh   = (DM_Plex *)dm->data;
4263     PetscInt  degree = 1, deg;
4264     PetscInt  height = 0;
4265     DM        cdm;
4266     PetscBool flg;
4267 
4268     PetscCall(PetscOptionsBool("-dm_coord_space", "Use an FEM space for coordinates", "", coordSpace, &coordSpace, &flg));
4269     PetscCall(PetscOptionsInt("-dm_coord_petscspace_degree", "FEM degree for coordinate space", "", degree, &degree, NULL));
4270     PetscCall(DMGetCoordinateDegree_Internal(dm, &deg));
4271     if (coordSpace && deg <= 1) PetscCall(DMPlexCreateCoordinateSpace(dm, degree, PETSC_TRUE, mesh->coordFunc));
4272     PetscCall(DMGetCoordinateDM(dm, &cdm));
4273     if (flg && !coordSpace) {
4274       PetscDS      cds;
4275       PetscObject  obj;
4276       PetscClassId id;
4277 
4278       PetscCall(DMGetDS(cdm, &cds));
4279       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
4280       PetscCall(PetscObjectGetClassId(obj, &id));
4281       if (id == PETSCFE_CLASSID) {
4282         PetscContainer dummy;
4283 
4284         PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &dummy));
4285         PetscCall(PetscObjectSetName((PetscObject)dummy, "coordinates"));
4286         PetscCall(DMSetField(cdm, 0, NULL, (PetscObject)dummy));
4287         PetscCall(PetscContainerDestroy(&dummy));
4288         PetscCall(DMClearDS(cdm));
4289       }
4290       mesh->coordFunc = NULL;
4291     }
4292     PetscCall(PetscOptionsBool("-dm_sparse_localize", "Localize only necessary cells", "", dm->sparseLocalize, &dm->sparseLocalize, &flg));
4293     PetscCall(PetscOptionsInt("-dm_localize_height", "Localize edges and faces in addition to cells", "", height, &height, &flg));
4294     if (flg) PetscCall(DMPlexSetMaxProjectionHeight(cdm, height));
4295     PetscCall(DMLocalizeCoordinates(dm));
4296   }
4297   /* Handle DMPlex refinement */
4298   remap = PETSC_TRUE;
4299   PetscCall(PetscOptionsBoundedInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL, 0));
4300   PetscCall(PetscOptionsBool("-dm_refine_remap", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
4301   PetscCall(PetscOptionsBoundedInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy, 0));
4302   if (refine) PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
4303   if (refine && isHierarchy) {
4304     DM *dms, coarseDM;
4305 
4306     PetscCall(DMGetCoarseDM(dm, &coarseDM));
4307     PetscCall(PetscObjectReference((PetscObject)coarseDM));
4308     PetscCall(PetscMalloc1(refine, &dms));
4309     PetscCall(DMRefineHierarchy(dm, refine, dms));
4310     /* Total hack since we do not pass in a pointer */
4311     PetscCall(DMPlexSwap_Static(dm, dms[refine - 1]));
4312     if (refine == 1) {
4313       PetscCall(DMSetCoarseDM(dm, dms[0]));
4314       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4315     } else {
4316       PetscCall(DMSetCoarseDM(dm, dms[refine - 2]));
4317       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4318       PetscCall(DMSetCoarseDM(dms[0], dms[refine - 1]));
4319       PetscCall(DMPlexSetRegularRefinement(dms[0], PETSC_TRUE));
4320     }
4321     PetscCall(DMSetCoarseDM(dms[refine - 1], coarseDM));
4322     PetscCall(PetscObjectDereference((PetscObject)coarseDM));
4323     /* Free DMs */
4324     for (r = 0; r < refine; ++r) {
4325       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4326       PetscCall(DMDestroy(&dms[r]));
4327     }
4328     PetscCall(PetscFree(dms));
4329   } else {
4330     for (r = 0; r < refine; ++r) {
4331       DM             rdm;
4332       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4333 
4334       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4335       PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
4336       /* Total hack since we do not pass in a pointer */
4337       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4338       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4339       if (coordFunc && remap) {
4340         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4341         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4342       }
4343     }
4344   }
4345   /* Handle DMPlex coarsening */
4346   PetscCall(PetscOptionsBoundedInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL, 0));
4347   PetscCall(PetscOptionsBoundedInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy, 0));
4348   if (coarsen && isHierarchy) {
4349     DM *dms;
4350 
4351     PetscCall(PetscMalloc1(coarsen, &dms));
4352     PetscCall(DMCoarsenHierarchy(dm, coarsen, dms));
4353     /* Free DMs */
4354     for (r = 0; r < coarsen; ++r) {
4355       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4356       PetscCall(DMDestroy(&dms[r]));
4357     }
4358     PetscCall(PetscFree(dms));
4359   } else {
4360     for (r = 0; r < coarsen; ++r) {
4361       DM             cdm;
4362       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4363 
4364       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4365       PetscCall(DMCoarsen(dm, PetscObjectComm((PetscObject)dm), &cdm));
4366       /* Total hack since we do not pass in a pointer */
4367       PetscCall(DMPlexReplace_Internal(dm, &cdm));
4368       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4369       if (coordFunc) {
4370         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4371         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4372       }
4373     }
4374   }
4375   /* Handle ghost cells */
4376   PetscCall(PetscOptionsBool("-dm_plex_create_fv_ghost_cells", "Flag to create finite volume ghost cells on the boundary", "DMCreate", ghostCells, &ghostCells, NULL));
4377   if (ghostCells) {
4378     DM   gdm;
4379     char lname[PETSC_MAX_PATH_LEN];
4380 
4381     lname[0] = '\0';
4382     PetscCall(PetscOptionsString("-dm_plex_fv_ghost_cells_label", "Label name for ghost cells boundary", "DMCreate", lname, lname, sizeof(lname), &flg));
4383     PetscCall(DMPlexConstructGhostCells(dm, flg ? lname : NULL, NULL, &gdm));
4384     PetscCall(DMPlexReplace_Internal(dm, &gdm));
4385   }
4386   /* Handle 1D order */
4387   if (reorder != DMPLEX_REORDER_DEFAULT_FALSE && dim == 1) {
4388     DM           cdm, rdm;
4389     PetscDS      cds;
4390     PetscObject  obj;
4391     PetscClassId id = PETSC_OBJECT_CLASSID;
4392     IS           perm;
4393     PetscInt     Nf;
4394     PetscBool    distributed;
4395 
4396     PetscCall(DMPlexIsDistributed(dm, &distributed));
4397     PetscCall(DMGetCoordinateDM(dm, &cdm));
4398     PetscCall(DMGetDS(cdm, &cds));
4399     PetscCall(PetscDSGetNumFields(cds, &Nf));
4400     if (Nf) {
4401       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
4402       PetscCall(PetscObjectGetClassId(obj, &id));
4403     }
4404     if (!distributed && id != PETSCFE_CLASSID) {
4405       PetscCall(DMPlexGetOrdering1D(dm, &perm));
4406       PetscCall(DMPlexPermute(dm, perm, &rdm));
4407       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4408       PetscCall(ISDestroy(&perm));
4409     }
4410   }
4411 /* Handle */
4412 non_refine:
4413   PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4414   PetscOptionsHeadEnd();
4415   PetscFunctionReturn(PETSC_SUCCESS);
4416 }
4417 
4418 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm, Vec *vec)
4419 {
4420   PetscFunctionBegin;
4421   PetscCall(DMCreateGlobalVector_Section_Private(dm, vec));
4422   /* PetscCall(VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM)); */
4423   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex));
4424   PetscCall(VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void))VecView_Plex_Native));
4425   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex));
4426   PetscCall(VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void))VecLoad_Plex_Native));
4427   PetscFunctionReturn(PETSC_SUCCESS);
4428 }
4429 
4430 static PetscErrorCode DMCreateLocalVector_Plex(DM dm, Vec *vec)
4431 {
4432   PetscFunctionBegin;
4433   PetscCall(DMCreateLocalVector_Section_Private(dm, vec));
4434   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex_Local));
4435   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex_Local));
4436   PetscFunctionReturn(PETSC_SUCCESS);
4437 }
4438 
4439 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
4440 {
4441   PetscInt depth, d;
4442 
4443   PetscFunctionBegin;
4444   PetscCall(DMPlexGetDepth(dm, &depth));
4445   if (depth == 1) {
4446     PetscCall(DMGetDimension(dm, &d));
4447     if (dim == 0) PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4448     else if (dim == d) PetscCall(DMPlexGetDepthStratum(dm, 1, pStart, pEnd));
4449     else {
4450       *pStart = 0;
4451       *pEnd   = 0;
4452     }
4453   } else {
4454     PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4455   }
4456   PetscFunctionReturn(PETSC_SUCCESS);
4457 }
4458 
4459 static PetscErrorCode DMGetNeighbors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
4460 {
4461   PetscSF            sf;
4462   PetscInt           niranks, njranks, n;
4463   const PetscMPIInt *iranks, *jranks;
4464   DM_Plex           *data = (DM_Plex *)dm->data;
4465 
4466   PetscFunctionBegin;
4467   PetscCall(DMGetPointSF(dm, &sf));
4468   if (!data->neighbors) {
4469     PetscCall(PetscSFSetUp(sf));
4470     PetscCall(PetscSFGetRootRanks(sf, &njranks, &jranks, NULL, NULL, NULL));
4471     PetscCall(PetscSFGetLeafRanks(sf, &niranks, &iranks, NULL, NULL));
4472     PetscCall(PetscMalloc1(njranks + niranks + 1, &data->neighbors));
4473     PetscCall(PetscArraycpy(data->neighbors + 1, jranks, njranks));
4474     PetscCall(PetscArraycpy(data->neighbors + njranks + 1, iranks, niranks));
4475     n = njranks + niranks;
4476     PetscCall(PetscSortRemoveDupsMPIInt(&n, data->neighbors + 1));
4477     /* The following cast should never fail: can't have more neighbors than PETSC_MPI_INT_MAX */
4478     PetscCall(PetscMPIIntCast(n, data->neighbors));
4479   }
4480   if (nranks) *nranks = data->neighbors[0];
4481   if (ranks) {
4482     if (data->neighbors[0]) *ranks = data->neighbors + 1;
4483     else *ranks = NULL;
4484   }
4485   PetscFunctionReturn(PETSC_SUCCESS);
4486 }
4487 
4488 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM, DM, Mat, Vec, Vec);
4489 
4490 static PetscErrorCode DMInitialize_Plex(DM dm)
4491 {
4492   PetscFunctionBegin;
4493   dm->ops->view                      = DMView_Plex;
4494   dm->ops->load                      = DMLoad_Plex;
4495   dm->ops->setfromoptions            = DMSetFromOptions_Plex;
4496   dm->ops->clone                     = DMClone_Plex;
4497   dm->ops->setup                     = DMSetUp_Plex;
4498   dm->ops->createlocalsection        = DMCreateLocalSection_Plex;
4499   dm->ops->createdefaultconstraints  = DMCreateDefaultConstraints_Plex;
4500   dm->ops->createglobalvector        = DMCreateGlobalVector_Plex;
4501   dm->ops->createlocalvector         = DMCreateLocalVector_Plex;
4502   dm->ops->getlocaltoglobalmapping   = NULL;
4503   dm->ops->createfieldis             = NULL;
4504   dm->ops->createcoordinatedm        = DMCreateCoordinateDM_Plex;
4505   dm->ops->createcoordinatefield     = DMCreateCoordinateField_Plex;
4506   dm->ops->getcoloring               = NULL;
4507   dm->ops->creatematrix              = DMCreateMatrix_Plex;
4508   dm->ops->createinterpolation       = DMCreateInterpolation_Plex;
4509   dm->ops->createmassmatrix          = DMCreateMassMatrix_Plex;
4510   dm->ops->createmassmatrixlumped    = DMCreateMassMatrixLumped_Plex;
4511   dm->ops->createinjection           = DMCreateInjection_Plex;
4512   dm->ops->refine                    = DMRefine_Plex;
4513   dm->ops->coarsen                   = DMCoarsen_Plex;
4514   dm->ops->refinehierarchy           = DMRefineHierarchy_Plex;
4515   dm->ops->coarsenhierarchy          = DMCoarsenHierarchy_Plex;
4516   dm->ops->extrude                   = DMExtrude_Plex;
4517   dm->ops->globaltolocalbegin        = NULL;
4518   dm->ops->globaltolocalend          = NULL;
4519   dm->ops->localtoglobalbegin        = NULL;
4520   dm->ops->localtoglobalend          = NULL;
4521   dm->ops->destroy                   = DMDestroy_Plex;
4522   dm->ops->createsubdm               = DMCreateSubDM_Plex;
4523   dm->ops->createsuperdm             = DMCreateSuperDM_Plex;
4524   dm->ops->getdimpoints              = DMGetDimPoints_Plex;
4525   dm->ops->locatepoints              = DMLocatePoints_Plex;
4526   dm->ops->projectfunctionlocal      = DMProjectFunctionLocal_Plex;
4527   dm->ops->projectfunctionlabellocal = DMProjectFunctionLabelLocal_Plex;
4528   dm->ops->projectfieldlocal         = DMProjectFieldLocal_Plex;
4529   dm->ops->projectfieldlabellocal    = DMProjectFieldLabelLocal_Plex;
4530   dm->ops->projectbdfieldlabellocal  = DMProjectBdFieldLabelLocal_Plex;
4531   dm->ops->computel2diff             = DMComputeL2Diff_Plex;
4532   dm->ops->computel2gradientdiff     = DMComputeL2GradientDiff_Plex;
4533   dm->ops->computel2fielddiff        = DMComputeL2FieldDiff_Plex;
4534   dm->ops->getneighbors              = DMGetNeighbors_Plex;
4535   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertBoundaryValues_C", DMPlexInsertBoundaryValues_Plex));
4536   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertTimeDerviativeBoundaryValues_C", DMPlexInsertTimeDerivativeBoundaryValues_Plex));
4537   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMSetUpGLVisViewer_C", DMSetUpGLVisViewer_Plex));
4538   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMCreateNeumannOverlap_C", DMCreateNeumannOverlap_Plex));
4539   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4540   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeGetDefault_C", DMPlexDistributeGetDefault_Plex));
4541   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeSetDefault_C", DMPlexDistributeSetDefault_Plex));
4542   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderGetDefault_C", DMPlexReorderGetDefault_Plex));
4543   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderSetDefault_C", DMPlexReorderSetDefault_Plex));
4544   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMInterpolateSolution_C", DMInterpolateSolution_Plex));
4545   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4546   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetOverlap_C", DMPlexSetOverlap_Plex));
4547   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetUseCeed_C", DMPlexGetUseCeed_Plex));
4548   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetUseCeed_C", DMPlexSetUseCeed_Plex));
4549   PetscFunctionReturn(PETSC_SUCCESS);
4550 }
4551 
4552 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
4553 {
4554   DM_Plex *mesh = (DM_Plex *)dm->data;
4555   PetscSF  face_sf;
4556 
4557   PetscFunctionBegin;
4558   mesh->refct++;
4559   (*newdm)->data = mesh;
4560   PetscCall(DMPlexGetIsoperiodicFaceSF(dm, &face_sf));
4561   PetscCall(DMPlexSetIsoperiodicFaceSF(*newdm, face_sf));
4562   PetscCall(PetscObjectChangeTypeName((PetscObject)*newdm, DMPLEX));
4563   PetscCall(DMInitialize_Plex(*newdm));
4564   PetscFunctionReturn(PETSC_SUCCESS);
4565 }
4566 
4567 /*MC
4568   DMPLEX = "plex" - A `DM` object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
4569                     In the local representation, `Vec`s contain all unknowns in the interior and shared boundary. This is
4570                     specified by a PetscSection object. Ownership in the global representation is determined by
4571                     ownership of the underlying `DMPLEX` points. This is specified by another `PetscSection` object.
4572 
4573   Options Database Keys:
4574 + -dm_refine_pre                     - Refine mesh before distribution
4575 + -dm_refine_uniform_pre             - Choose uniform or generator-based refinement
4576 + -dm_refine_volume_limit_pre        - Cell volume limit after pre-refinement using generator
4577 . -dm_distribute                     - Distribute mesh across processes
4578 . -dm_distribute_overlap             - Number of cells to overlap for distribution
4579 . -dm_refine                         - Refine mesh after distribution
4580 . -dm_plex_hash_location             - Use grid hashing for point location
4581 . -dm_plex_hash_box_faces <n,m,p>    - The number of divisions in each direction of the grid hash
4582 . -dm_plex_partition_balance         - Attempt to evenly divide points on partition boundary between processes
4583 . -dm_plex_remesh_bd                 - Allow changes to the boundary on remeshing
4584 . -dm_plex_max_projection_height     - Maximum mesh point height used to project locally
4585 . -dm_plex_regular_refinement        - Use special nested projection algorithm for regular refinement
4586 . -dm_plex_check_all                 - Perform all checks below
4587 . -dm_plex_check_symmetry            - Check that the adjacency information in the mesh is symmetric
4588 . -dm_plex_check_skeleton <celltype> - Check that each cell has the correct number of vertices
4589 . -dm_plex_check_faces <celltype>    - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
4590 . -dm_plex_check_geometry            - Check that cells have positive volume
4591 . -dm_view :mesh.tex:ascii_latex     - View the mesh in LaTeX/TikZ
4592 . -dm_plex_view_scale <num>          - Scale the TikZ
4593 - -dm_plex_print_fem <num>           - View FEM assembly information, such as element vectors and matrices
4594 
4595   Level: intermediate
4596 
4597 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`, `PetscSection`
4598 M*/
4599 
4600 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
4601 {
4602   DM_Plex *mesh;
4603   PetscInt unit;
4604 
4605   PetscFunctionBegin;
4606   PetscCall(PetscCitationsRegister(PlexCitation, &Plexcite));
4607   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4608   PetscCall(PetscNew(&mesh));
4609   dm->data = mesh;
4610 
4611   mesh->refct = 1;
4612   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection));
4613   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection));
4614   mesh->refinementUniform      = PETSC_TRUE;
4615   mesh->refinementLimit        = -1.0;
4616   mesh->distDefault            = PETSC_TRUE;
4617   mesh->reorderDefault         = DMPLEX_REORDER_DEFAULT_NOTSET;
4618   mesh->distributionName       = NULL;
4619   mesh->interpolated           = DMPLEX_INTERPOLATED_INVALID;
4620   mesh->interpolatedCollective = DMPLEX_INTERPOLATED_INVALID;
4621 
4622   PetscCall(PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner));
4623   mesh->remeshBd = PETSC_FALSE;
4624 
4625   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
4626 
4627   mesh->depthState    = -1;
4628   mesh->celltypeState = -1;
4629   mesh->printTol      = 1.0e-10;
4630 
4631   PetscCall(DMInitialize_Plex(dm));
4632   PetscFunctionReturn(PETSC_SUCCESS);
4633 }
4634 
4635 /*@
4636   DMPlexCreate - Creates a `DMPLEX` object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
4637 
4638   Collective
4639 
4640   Input Parameter:
4641 . comm - The communicator for the `DMPLEX` object
4642 
4643   Output Parameter:
4644 . mesh - The `DMPLEX` object
4645 
4646   Level: beginner
4647 
4648 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMType`, `DMCreate()`, `DMSetType()`
4649 @*/
4650 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
4651 {
4652   PetscFunctionBegin;
4653   PetscAssertPointer(mesh, 2);
4654   PetscCall(DMCreate(comm, mesh));
4655   PetscCall(DMSetType(*mesh, DMPLEX));
4656   PetscFunctionReturn(PETSC_SUCCESS);
4657 }
4658 
4659 /*@C
4660   DMPlexBuildFromCellListParallel - Build distributed `DMPLEX` topology from a list of vertices for each cell (common mesh generator output)
4661 
4662   Collective; No Fortran Support
4663 
4664   Input Parameters:
4665 + dm          - The `DM`
4666 . numCells    - The number of cells owned by this process
4667 . numVertices - The number of vertices to be owned by this process, or `PETSC_DECIDE`
4668 . NVertices   - The global number of vertices, or `PETSC_DETERMINE`
4669 . numCorners  - The number of vertices for each cell
4670 - cells       - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4671 
4672   Output Parameters:
4673 + vertexSF         - (Optional) `PetscSF` describing complete vertex ownership
4674 - verticesAdjSaved - (Optional) vertex adjacency array
4675 
4676   Level: advanced
4677 
4678   Notes:
4679   Two triangles sharing a face
4680 .vb
4681 
4682         2
4683       / | \
4684      /  |  \
4685     /   |   \
4686    0  0 | 1  3
4687     \   |   /
4688      \  |  /
4689       \ | /
4690         1
4691 .ve
4692   would have input
4693 .vb
4694   numCells = 2, numVertices = 4
4695   cells = [0 1 2  1 3 2]
4696 .ve
4697   which would result in the `DMPLEX`
4698 .vb
4699 
4700         4
4701       / | \
4702      /  |  \
4703     /   |   \
4704    2  0 | 1  5
4705     \   |   /
4706      \  |  /
4707       \ | /
4708         3
4709 .ve
4710 
4711   Vertices are implicitly numbered consecutively 0,...,NVertices.
4712   Each rank owns a chunk of numVertices consecutive vertices.
4713   If numVertices is `PETSC_DECIDE`, PETSc will distribute them as evenly as possible using PetscLayout.
4714   If NVertices is `PETSC_DETERMINE` and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
4715   If only NVertices is `PETSC_DETERMINE`, it is computed as the sum of numVertices over all ranks.
4716 
4717   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
4718 
4719 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildCoordinatesFromCellListParallel()`,
4720           `PetscSF`
4721 @*/
4722 PetscErrorCode DMPlexBuildFromCellListParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, const PetscInt cells[], PetscSF *vertexSF, PetscInt **verticesAdjSaved)
4723 {
4724   PetscSF     sfPoint;
4725   PetscLayout layout;
4726   PetscInt    numVerticesAdj, *verticesAdj, *cones, c, p;
4727 
4728   PetscFunctionBegin;
4729   PetscValidLogicalCollectiveInt(dm, NVertices, 4);
4730   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4731   /* Get/check global number of vertices */
4732   {
4733     PetscInt       NVerticesInCells, i;
4734     const PetscInt len = numCells * numCorners;
4735 
4736     /* NVerticesInCells = max(cells) + 1 */
4737     NVerticesInCells = PETSC_MIN_INT;
4738     for (i = 0; i < len; i++)
4739       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4740     ++NVerticesInCells;
4741     PetscCall(MPIU_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
4742 
4743     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
4744     else
4745       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
4746   }
4747   /* Count locally unique vertices */
4748   {
4749     PetscHSetI vhash;
4750     PetscInt   off = 0;
4751 
4752     PetscCall(PetscHSetICreate(&vhash));
4753     for (c = 0; c < numCells; ++c) {
4754       for (p = 0; p < numCorners; ++p) PetscCall(PetscHSetIAdd(vhash, cells[c * numCorners + p]));
4755     }
4756     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
4757     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
4758     else verticesAdj = *verticesAdjSaved;
4759     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
4760     PetscCall(PetscHSetIDestroy(&vhash));
4761     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
4762   }
4763   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
4764   /* Create cones */
4765   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
4766   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
4767   PetscCall(DMSetUp(dm));
4768   PetscCall(DMPlexGetCones(dm, &cones));
4769   for (c = 0; c < numCells; ++c) {
4770     for (p = 0; p < numCorners; ++p) {
4771       const PetscInt gv = cells[c * numCorners + p];
4772       PetscInt       lv;
4773 
4774       /* Positions within verticesAdj form 0-based local vertex numbering;
4775          we need to shift it by numCells to get correct DAG points (cells go first) */
4776       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
4777       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
4778       cones[c * numCorners + p] = lv + numCells;
4779     }
4780   }
4781   /* Build point sf */
4782   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
4783   PetscCall(PetscLayoutSetSize(layout, NVertices));
4784   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
4785   PetscCall(PetscLayoutSetBlockSize(layout, 1));
4786   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
4787   PetscCall(PetscLayoutDestroy(&layout));
4788   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
4789   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
4790   if (dm->sf) {
4791     const char *prefix;
4792 
4793     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
4794     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
4795   }
4796   PetscCall(DMSetPointSF(dm, sfPoint));
4797   PetscCall(PetscSFDestroy(&sfPoint));
4798   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)(*vertexSF), "Vertex Ownership SF"));
4799   /* Fill in the rest of the topology structure */
4800   PetscCall(DMPlexSymmetrize(dm));
4801   PetscCall(DMPlexStratify(dm));
4802   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4803   PetscFunctionReturn(PETSC_SUCCESS);
4804 }
4805 
4806 /*@C
4807   DMPlexBuildCoordinatesFromCellListParallel - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4808 
4809   Collective; No Fortran Support
4810 
4811   Input Parameters:
4812 + dm           - The `DM`
4813 . spaceDim     - The spatial dimension used for coordinates
4814 . sfVert       - `PetscSF` describing complete vertex ownership
4815 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4816 
4817   Level: advanced
4818 
4819 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellListParallel()`
4820 @*/
4821 PetscErrorCode DMPlexBuildCoordinatesFromCellListParallel(DM dm, PetscInt spaceDim, PetscSF sfVert, const PetscReal vertexCoords[])
4822 {
4823   PetscSection coordSection;
4824   Vec          coordinates;
4825   PetscScalar *coords;
4826   PetscInt     numVertices, numVerticesAdj, coordSize, v, vStart, vEnd;
4827 
4828   PetscFunctionBegin;
4829   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4830   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
4831   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4832   PetscCall(DMSetCoordinateDim(dm, spaceDim));
4833   PetscCall(PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL));
4834   PetscCheck(vEnd - vStart == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Supplied sfVert has wrong number of leaves = %" PetscInt_FMT " != %" PetscInt_FMT " = vEnd - vStart", numVerticesAdj, vEnd - vStart);
4835   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4836   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4837   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
4838   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
4839   for (v = vStart; v < vEnd; ++v) {
4840     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
4841     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
4842   }
4843   PetscCall(PetscSectionSetUp(coordSection));
4844   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4845   PetscCall(VecCreate(PetscObjectComm((PetscObject)dm), &coordinates));
4846   PetscCall(VecSetBlockSize(coordinates, spaceDim));
4847   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4848   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4849   PetscCall(VecSetType(coordinates, VECSTANDARD));
4850   PetscCall(VecGetArray(coordinates, &coords));
4851   {
4852     MPI_Datatype coordtype;
4853 
4854     /* Need a temp buffer for coords if we have complex/single */
4855     PetscCallMPI(MPI_Type_contiguous(spaceDim, MPIU_SCALAR, &coordtype));
4856     PetscCallMPI(MPI_Type_commit(&coordtype));
4857 #if defined(PETSC_USE_COMPLEX)
4858     {
4859       PetscScalar *svertexCoords;
4860       PetscInt     i;
4861       PetscCall(PetscMalloc1(numVertices * spaceDim, &svertexCoords));
4862       for (i = 0; i < numVertices * spaceDim; i++) svertexCoords[i] = vertexCoords[i];
4863       PetscCall(PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4864       PetscCall(PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4865       PetscCall(PetscFree(svertexCoords));
4866     }
4867 #else
4868     PetscCall(PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4869     PetscCall(PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4870 #endif
4871     PetscCallMPI(MPI_Type_free(&coordtype));
4872   }
4873   PetscCall(VecRestoreArray(coordinates, &coords));
4874   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4875   PetscCall(VecDestroy(&coordinates));
4876   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4877   PetscFunctionReturn(PETSC_SUCCESS);
4878 }
4879 
4880 /*@
4881   DMPlexCreateFromCellListParallelPetsc - Create distributed `DMPLEX` from a list of vertices for each cell (common mesh generator output)
4882 
4883   Collective
4884 
4885   Input Parameters:
4886 + comm         - The communicator
4887 . dim          - The topological dimension of the mesh
4888 . numCells     - The number of cells owned by this process
4889 . numVertices  - The number of vertices owned by this process, or `PETSC_DECIDE`
4890 . NVertices    - The global number of vertices, or `PETSC_DECIDE`
4891 . numCorners   - The number of vertices for each cell
4892 . interpolate  - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4893 . cells        - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4894 . spaceDim     - The spatial dimension used for coordinates
4895 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4896 
4897   Output Parameters:
4898 + dm          - The `DM`
4899 . vertexSF    - (Optional) `PetscSF` describing complete vertex ownership
4900 - verticesAdj - (Optional) vertex adjacency array
4901 
4902   Level: intermediate
4903 
4904   Notes:
4905   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`,
4906   `DMPlexBuildFromCellListParallel()`, `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellListParallel()`
4907 
4908   See `DMPlexBuildFromCellListParallel()` for an example and details about the topology-related parameters.
4909 
4910   See `DMPlexBuildCoordinatesFromCellListParallel()` for details about the geometry-related parameters.
4911 
4912 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
4913 @*/
4914 PetscErrorCode DMPlexCreateFromCellListParallelPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, PetscInt **verticesAdj, DM *dm)
4915 {
4916   PetscSF sfVert;
4917 
4918   PetscFunctionBegin;
4919   PetscCall(DMCreate(comm, dm));
4920   PetscCall(DMSetType(*dm, DMPLEX));
4921   PetscValidLogicalCollectiveInt(*dm, dim, 2);
4922   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
4923   PetscCall(DMSetDimension(*dm, dim));
4924   PetscCall(DMPlexBuildFromCellListParallel(*dm, numCells, numVertices, NVertices, numCorners, cells, &sfVert, verticesAdj));
4925   if (interpolate) {
4926     DM idm;
4927 
4928     PetscCall(DMPlexInterpolate(*dm, &idm));
4929     PetscCall(DMDestroy(dm));
4930     *dm = idm;
4931   }
4932   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
4933   if (vertexSF) *vertexSF = sfVert;
4934   else PetscCall(PetscSFDestroy(&sfVert));
4935   PetscFunctionReturn(PETSC_SUCCESS);
4936 }
4937 
4938 /*@C
4939   DMPlexBuildFromCellList - Build `DMPLEX` topology from a list of vertices for each cell (common mesh generator output)
4940 
4941   Collective; No Fortran Support
4942 
4943   Input Parameters:
4944 + dm          - The `DM`
4945 . numCells    - The number of cells owned by this process
4946 . numVertices - The number of vertices owned by this process, or `PETSC_DETERMINE`
4947 . numCorners  - The number of vertices for each cell
4948 - cells       - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4949 
4950   Level: advanced
4951 
4952   Notes:
4953   Two triangles sharing a face
4954 .vb
4955 
4956         2
4957       / | \
4958      /  |  \
4959     /   |   \
4960    0  0 | 1  3
4961     \   |   /
4962      \  |  /
4963       \ | /
4964         1
4965 .ve
4966   would have input
4967 .vb
4968   numCells = 2, numVertices = 4
4969   cells = [0 1 2  1 3 2]
4970 .ve
4971   which would result in the `DMPLEX`
4972 .vb
4973 
4974         4
4975       / | \
4976      /  |  \
4977     /   |   \
4978    2  0 | 1  5
4979     \   |   /
4980      \  |  /
4981       \ | /
4982         3
4983 .ve
4984 
4985   If numVertices is `PETSC_DETERMINE`, it is computed by PETSc as the maximum vertex index in cells + 1.
4986 
4987 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListPetsc()`
4988 @*/
4989 PetscErrorCode DMPlexBuildFromCellList(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const PetscInt cells[])
4990 {
4991   PetscInt *cones, c, p, dim;
4992 
4993   PetscFunctionBegin;
4994   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4995   PetscCall(DMGetDimension(dm, &dim));
4996   /* Get/check global number of vertices */
4997   {
4998     PetscInt       NVerticesInCells, i;
4999     const PetscInt len = numCells * numCorners;
5000 
5001     /* NVerticesInCells = max(cells) + 1 */
5002     NVerticesInCells = PETSC_MIN_INT;
5003     for (i = 0; i < len; i++)
5004       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
5005     ++NVerticesInCells;
5006 
5007     if (numVertices == PETSC_DECIDE) numVertices = NVerticesInCells;
5008     else
5009       PetscCheck(numVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, numVertices, NVerticesInCells);
5010   }
5011   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
5012   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
5013   PetscCall(DMSetUp(dm));
5014   PetscCall(DMPlexGetCones(dm, &cones));
5015   for (c = 0; c < numCells; ++c) {
5016     for (p = 0; p < numCorners; ++p) cones[c * numCorners + p] = cells[c * numCorners + p] + numCells;
5017   }
5018   PetscCall(DMPlexSymmetrize(dm));
5019   PetscCall(DMPlexStratify(dm));
5020   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5021   PetscFunctionReturn(PETSC_SUCCESS);
5022 }
5023 
5024 /*@C
5025   DMPlexBuildCoordinatesFromCellList - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
5026 
5027   Collective; No Fortran Support
5028 
5029   Input Parameters:
5030 + dm           - The `DM`
5031 . spaceDim     - The spatial dimension used for coordinates
5032 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
5033 
5034   Level: advanced
5035 
5036 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellList()`
5037 @*/
5038 PetscErrorCode DMPlexBuildCoordinatesFromCellList(DM dm, PetscInt spaceDim, const PetscReal vertexCoords[])
5039 {
5040   PetscSection coordSection;
5041   Vec          coordinates;
5042   DM           cdm;
5043   PetscScalar *coords;
5044   PetscInt     v, vStart, vEnd, d;
5045 
5046   PetscFunctionBegin;
5047   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
5048   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
5049   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
5050   PetscCall(DMSetCoordinateDim(dm, spaceDim));
5051   PetscCall(DMGetCoordinateSection(dm, &coordSection));
5052   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5053   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
5054   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
5055   for (v = vStart; v < vEnd; ++v) {
5056     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
5057     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
5058   }
5059   PetscCall(PetscSectionSetUp(coordSection));
5060 
5061   PetscCall(DMGetCoordinateDM(dm, &cdm));
5062   PetscCall(DMCreateLocalVector(cdm, &coordinates));
5063   PetscCall(VecSetBlockSize(coordinates, spaceDim));
5064   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5065   PetscCall(VecGetArrayWrite(coordinates, &coords));
5066   for (v = 0; v < vEnd - vStart; ++v) {
5067     for (d = 0; d < spaceDim; ++d) coords[v * spaceDim + d] = vertexCoords[v * spaceDim + d];
5068   }
5069   PetscCall(VecRestoreArrayWrite(coordinates, &coords));
5070   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
5071   PetscCall(VecDestroy(&coordinates));
5072   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
5073   PetscFunctionReturn(PETSC_SUCCESS);
5074 }
5075 
5076 /*@
5077   DMPlexCreateFromCellListPetsc - Create `DMPLEX` from a list of vertices for each cell (common mesh generator output), but only process 0 takes in the input
5078 
5079   Collective
5080 
5081   Input Parameters:
5082 + comm         - The communicator
5083 . dim          - The topological dimension of the mesh
5084 . numCells     - The number of cells, only on process 0
5085 . numVertices  - The number of vertices owned by this process, or `PETSC_DECIDE`, only on process 0
5086 . numCorners   - The number of vertices for each cell, only on process 0
5087 . interpolate  - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
5088 . cells        - An array of numCells*numCorners numbers, the vertices for each cell, only on process 0
5089 . spaceDim     - The spatial dimension used for coordinates
5090 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex, only on process 0
5091 
5092   Output Parameter:
5093 . dm - The `DM`, which only has points on process 0
5094 
5095   Level: intermediate
5096 
5097   Notes:
5098   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`, `DMPlexBuildFromCellList()`,
5099   `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellList()`
5100 
5101   See `DMPlexBuildFromCellList()` for an example and details about the topology-related parameters.
5102   See `DMPlexBuildCoordinatesFromCellList()` for details about the geometry-related parameters.
5103   See `DMPlexCreateFromCellListParallelPetsc()` for parallel input
5104 
5105 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellList()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
5106 @*/
5107 PetscErrorCode DMPlexCreateFromCellListPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], DM *dm)
5108 {
5109   PetscMPIInt rank;
5110 
5111   PetscFunctionBegin;
5112   PetscCheck(dim, comm, PETSC_ERR_ARG_OUTOFRANGE, "This is not appropriate for 0-dimensional meshes. Consider either creating the DM using DMPlexCreateFromDAG(), by hand, or using DMSwarm.");
5113   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5114   PetscCall(DMCreate(comm, dm));
5115   PetscCall(DMSetType(*dm, DMPLEX));
5116   PetscCall(DMSetDimension(*dm, dim));
5117   if (rank == 0) PetscCall(DMPlexBuildFromCellList(*dm, numCells, numVertices, numCorners, cells));
5118   else PetscCall(DMPlexBuildFromCellList(*dm, 0, 0, 0, NULL));
5119   if (interpolate) {
5120     DM idm;
5121 
5122     PetscCall(DMPlexInterpolate(*dm, &idm));
5123     PetscCall(DMDestroy(dm));
5124     *dm = idm;
5125   }
5126   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, vertexCoords));
5127   else PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, NULL));
5128   PetscFunctionReturn(PETSC_SUCCESS);
5129 }
5130 
5131 /*@
5132   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a `DM`
5133 
5134   Input Parameters:
5135 + dm               - The empty `DM` object, usually from `DMCreate()` and `DMSetDimension()`
5136 . depth            - The depth of the DAG
5137 . numPoints        - Array of size depth + 1 containing the number of points at each `depth`
5138 . coneSize         - The cone size of each point
5139 . cones            - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
5140 . coneOrientations - The orientation of each cone point
5141 - vertexCoords     - An array of `numPoints`[0]*spacedim numbers representing the coordinates of each vertex, with spacedim the value set via `DMSetCoordinateDim()`
5142 
5143   Output Parameter:
5144 . dm - The `DM`
5145 
5146   Level: advanced
5147 
5148   Note:
5149   Two triangles sharing a face would have input
5150 .vb
5151   depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
5152   cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
5153  vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
5154 .ve
5155   which would result in the DMPlex
5156 .vb
5157         4
5158       / | \
5159      /  |  \
5160     /   |   \
5161    2  0 | 1  5
5162     \   |   /
5163      \  |  /
5164       \ | /
5165         3
5166 .ve
5167   Notice that all points are numbered consecutively, unlike `DMPlexCreateFromCellListPetsc()`
5168 
5169 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
5170 @*/
5171 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
5172 {
5173   Vec          coordinates;
5174   PetscSection coordSection;
5175   PetscScalar *coords;
5176   PetscInt     coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
5177 
5178   PetscFunctionBegin;
5179   PetscCall(DMGetDimension(dm, &dim));
5180   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
5181   PetscCheck(dimEmbed >= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Embedding dimension %" PetscInt_FMT " cannot be less than intrinsic dimension %" PetscInt_FMT, dimEmbed, dim);
5182   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
5183   PetscCall(DMPlexSetChart(dm, pStart, pEnd));
5184   for (p = pStart; p < pEnd; ++p) {
5185     PetscCall(DMPlexSetConeSize(dm, p, coneSize[p - pStart]));
5186     if (firstVertex < 0 && !coneSize[p - pStart]) firstVertex = p - pStart;
5187   }
5188   PetscCheck(firstVertex >= 0 || !numPoints[0], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Expected %" PetscInt_FMT " vertices but could not find any", numPoints[0]);
5189   PetscCall(DMSetUp(dm)); /* Allocate space for cones */
5190   for (p = pStart, off = 0; p < pEnd; off += coneSize[p - pStart], ++p) {
5191     PetscCall(DMPlexSetCone(dm, p, &cones[off]));
5192     PetscCall(DMPlexSetConeOrientation(dm, p, &coneOrientations[off]));
5193   }
5194   PetscCall(DMPlexSymmetrize(dm));
5195   PetscCall(DMPlexStratify(dm));
5196   /* Build coordinates */
5197   PetscCall(DMGetCoordinateSection(dm, &coordSection));
5198   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5199   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dimEmbed));
5200   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numPoints[0]));
5201   for (v = firstVertex; v < firstVertex + numPoints[0]; ++v) {
5202     PetscCall(PetscSectionSetDof(coordSection, v, dimEmbed));
5203     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed));
5204   }
5205   PetscCall(PetscSectionSetUp(coordSection));
5206   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
5207   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
5208   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5209   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
5210   PetscCall(VecSetBlockSize(coordinates, dimEmbed));
5211   PetscCall(VecSetType(coordinates, VECSTANDARD));
5212   if (vertexCoords) {
5213     PetscCall(VecGetArray(coordinates, &coords));
5214     for (v = 0; v < numPoints[0]; ++v) {
5215       PetscInt off;
5216 
5217       PetscCall(PetscSectionGetOffset(coordSection, v + firstVertex, &off));
5218       for (d = 0; d < dimEmbed; ++d) coords[off + d] = vertexCoords[v * dimEmbed + d];
5219     }
5220   }
5221   PetscCall(VecRestoreArray(coordinates, &coords));
5222   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
5223   PetscCall(VecDestroy(&coordinates));
5224   PetscFunctionReturn(PETSC_SUCCESS);
5225 }
5226 
5227 /*
5228   DMPlexCreateCellVertexFromFile - Create a `DMPLEX` mesh from a simple cell-vertex file.
5229 
5230   Collective
5231 
5232 + comm        - The MPI communicator
5233 . filename    - Name of the .dat file
5234 - interpolate - Create faces and edges in the mesh
5235 
5236   Output Parameter:
5237 . dm  - The `DM` object representing the mesh
5238 
5239   Level: beginner
5240 
5241   Note:
5242   The format is the simplest possible:
5243 .vb
5244   Ne
5245   v0 v1 ... vk
5246   Nv
5247   x y z marker
5248 .ve
5249 
5250   Developer Note:
5251   Should use a `PetscViewer` not a filename
5252 
5253 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromFile()`, `DMPlexCreateMedFromFile()`, `DMPlexCreateGmsh()`, `DMPlexCreate()`
5254 */
5255 static PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
5256 {
5257   DMLabel      marker;
5258   PetscViewer  viewer;
5259   Vec          coordinates;
5260   PetscSection coordSection;
5261   PetscScalar *coords;
5262   char         line[PETSC_MAX_PATH_LEN];
5263   PetscInt     dim = 3, cdim = 3, coordSize, v, c, d;
5264   PetscMPIInt  rank;
5265   int          snum, Nv, Nc, Ncn, Nl;
5266 
5267   PetscFunctionBegin;
5268   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5269   PetscCall(PetscViewerCreate(comm, &viewer));
5270   PetscCall(PetscViewerSetType(viewer, PETSCVIEWERASCII));
5271   PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
5272   PetscCall(PetscViewerFileSetName(viewer, filename));
5273   if (rank == 0) {
5274     PetscCall(PetscViewerRead(viewer, line, 4, NULL, PETSC_STRING));
5275     snum = sscanf(line, "%d %d %d %d", &Nc, &Nv, &Ncn, &Nl);
5276     PetscCheck(snum == 4, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5277   } else {
5278     Nc = Nv = Ncn = Nl = 0;
5279   }
5280   PetscCall(DMCreate(comm, dm));
5281   PetscCall(DMSetType(*dm, DMPLEX));
5282   PetscCall(DMPlexSetChart(*dm, 0, Nc + Nv));
5283   PetscCall(DMSetDimension(*dm, dim));
5284   PetscCall(DMSetCoordinateDim(*dm, cdim));
5285   /* Read topology */
5286   if (rank == 0) {
5287     char     format[PETSC_MAX_PATH_LEN];
5288     PetscInt cone[8];
5289     int      vbuf[8], v;
5290 
5291     for (c = 0; c < Ncn; ++c) {
5292       format[c * 3 + 0] = '%';
5293       format[c * 3 + 1] = 'd';
5294       format[c * 3 + 2] = ' ';
5295     }
5296     format[Ncn * 3 - 1] = '\0';
5297     for (c = 0; c < Nc; ++c) PetscCall(DMPlexSetConeSize(*dm, c, Ncn));
5298     PetscCall(DMSetUp(*dm));
5299     for (c = 0; c < Nc; ++c) {
5300       PetscCall(PetscViewerRead(viewer, line, Ncn, NULL, PETSC_STRING));
5301       switch (Ncn) {
5302       case 2:
5303         snum = sscanf(line, format, &vbuf[0], &vbuf[1]);
5304         break;
5305       case 3:
5306         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2]);
5307         break;
5308       case 4:
5309         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3]);
5310         break;
5311       case 6:
5312         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5]);
5313         break;
5314       case 8:
5315         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);
5316         break;
5317       default:
5318         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No cell shape with %d vertices", Ncn);
5319       }
5320       PetscCheck(snum == Ncn, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5321       for (v = 0; v < Ncn; ++v) cone[v] = vbuf[v] + Nc;
5322       /* Hexahedra are inverted */
5323       if (Ncn == 8) {
5324         PetscInt tmp = cone[1];
5325         cone[1]      = cone[3];
5326         cone[3]      = tmp;
5327       }
5328       PetscCall(DMPlexSetCone(*dm, c, cone));
5329     }
5330   }
5331   PetscCall(DMPlexSymmetrize(*dm));
5332   PetscCall(DMPlexStratify(*dm));
5333   /* Read coordinates */
5334   PetscCall(DMGetCoordinateSection(*dm, &coordSection));
5335   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5336   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
5337   PetscCall(PetscSectionSetChart(coordSection, Nc, Nc + Nv));
5338   for (v = Nc; v < Nc + Nv; ++v) {
5339     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
5340     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
5341   }
5342   PetscCall(PetscSectionSetUp(coordSection));
5343   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
5344   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
5345   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5346   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
5347   PetscCall(VecSetBlockSize(coordinates, cdim));
5348   PetscCall(VecSetType(coordinates, VECSTANDARD));
5349   PetscCall(VecGetArray(coordinates, &coords));
5350   if (rank == 0) {
5351     char   format[PETSC_MAX_PATH_LEN];
5352     double x[3];
5353     int    l, val[3];
5354 
5355     if (Nl) {
5356       for (l = 0; l < Nl; ++l) {
5357         format[l * 3 + 0] = '%';
5358         format[l * 3 + 1] = 'd';
5359         format[l * 3 + 2] = ' ';
5360       }
5361       format[Nl * 3 - 1] = '\0';
5362       PetscCall(DMCreateLabel(*dm, "marker"));
5363       PetscCall(DMGetLabel(*dm, "marker", &marker));
5364     }
5365     for (v = 0; v < Nv; ++v) {
5366       PetscCall(PetscViewerRead(viewer, line, 3 + Nl, NULL, PETSC_STRING));
5367       snum = sscanf(line, "%lg %lg %lg", &x[0], &x[1], &x[2]);
5368       PetscCheck(snum == 3, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5369       switch (Nl) {
5370       case 0:
5371         snum = 0;
5372         break;
5373       case 1:
5374         snum = sscanf(line, format, &val[0]);
5375         break;
5376       case 2:
5377         snum = sscanf(line, format, &val[0], &val[1]);
5378         break;
5379       case 3:
5380         snum = sscanf(line, format, &val[0], &val[1], &val[2]);
5381         break;
5382       default:
5383         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Request support for %d labels", Nl);
5384       }
5385       PetscCheck(snum == Nl, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5386       for (d = 0; d < cdim; ++d) coords[v * cdim + d] = x[d];
5387       for (l = 0; l < Nl; ++l) PetscCall(DMLabelSetValue(marker, v + Nc, val[l]));
5388     }
5389   }
5390   PetscCall(VecRestoreArray(coordinates, &coords));
5391   PetscCall(DMSetCoordinatesLocal(*dm, coordinates));
5392   PetscCall(VecDestroy(&coordinates));
5393   PetscCall(PetscViewerDestroy(&viewer));
5394   if (interpolate) {
5395     DM      idm;
5396     DMLabel bdlabel;
5397 
5398     PetscCall(DMPlexInterpolate(*dm, &idm));
5399     PetscCall(DMDestroy(dm));
5400     *dm = idm;
5401 
5402     if (!Nl) {
5403       PetscCall(DMCreateLabel(*dm, "marker"));
5404       PetscCall(DMGetLabel(*dm, "marker", &bdlabel));
5405       PetscCall(DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel));
5406       PetscCall(DMPlexLabelComplete(*dm, bdlabel));
5407     }
5408   }
5409   PetscFunctionReturn(PETSC_SUCCESS);
5410 }
5411 
5412 /*@C
5413   DMPlexCreateFromFile - This takes a filename and produces a `DM`
5414 
5415   Collective
5416 
5417   Input Parameters:
5418 + comm        - The communicator
5419 . filename    - A file name
5420 . plexname    - The object name of the resulting `DM`, also used for intra-datafile lookup by some formats
5421 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
5422 
5423   Output Parameter:
5424 . dm - The `DM`
5425 
5426   Options Database Key:
5427 . -dm_plex_create_from_hdf5_xdmf - use the `PETSC_VIEWER_HDF5_XDMF` format for reading HDF5
5428 
5429   Use `-dm_plex_create_ prefix` to pass options to the internal `PetscViewer`, e.g.
5430 $ -dm_plex_create_viewer_hdf5_collective
5431 
5432   Level: beginner
5433 
5434   Notes:
5435   Using `PETSCVIEWERHDF5` type with `PETSC_VIEWER_HDF5_PETSC` format, one can save multiple `DMPLEX`
5436   meshes in a single HDF5 file. This in turn requires one to name the `DMPLEX` object with `PetscObjectSetName()`
5437   before saving it with `DMView()` and before loading it with `DMLoad()` for identification of the mesh object.
5438   The input parameter name is thus used to name the `DMPLEX` object when `DMPlexCreateFromFile()` internally
5439   calls `DMLoad()`. Currently, name is ignored for other viewer types and/or formats.
5440 
5441 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`, `PetscObjectSetName()`, `DMView()`, `DMLoad()`
5442 @*/
5443 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], const char plexname[], PetscBool interpolate, DM *dm)
5444 {
5445   const char  extGmsh[]      = ".msh";
5446   const char  extGmsh2[]     = ".msh2";
5447   const char  extGmsh4[]     = ".msh4";
5448   const char  extCGNS[]      = ".cgns";
5449   const char  extExodus[]    = ".exo";
5450   const char  extExodus_e[]  = ".e";
5451   const char  extGenesis[]   = ".gen";
5452   const char  extFluent[]    = ".cas";
5453   const char  extHDF5[]      = ".h5";
5454   const char  extXDMFHDF5[]  = ".xdmf.h5";
5455   const char  extMed[]       = ".med";
5456   const char  extPLY[]       = ".ply";
5457   const char  extEGADSLite[] = ".egadslite";
5458   const char  extEGADS[]     = ".egads";
5459   const char  extIGES[]      = ".igs";
5460   const char  extSTEP[]      = ".stp";
5461   const char  extCV[]        = ".dat";
5462   size_t      len;
5463   PetscBool   isGmsh, isGmsh2, isGmsh4, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isMed, isPLY, isEGADSLite, isEGADS, isIGES, isSTEP, isCV, isXDMFHDF5;
5464   PetscMPIInt rank;
5465 
5466   PetscFunctionBegin;
5467   PetscAssertPointer(filename, 2);
5468   if (plexname) PetscAssertPointer(plexname, 3);
5469   PetscAssertPointer(dm, 5);
5470   PetscCall(DMInitializePackage());
5471   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5472   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5473   PetscCall(PetscStrlen(filename, &len));
5474   PetscCheck(len, comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
5475 
5476 #define CheckExtension(extension__, is_extension__) \
5477   do { \
5478     PetscAssert(sizeof(extension__), comm, PETSC_ERR_PLIB, "Zero-size extension: %s", extension__); \
5479     /* don't count the null-terminator at the end */ \
5480     const size_t ext_len = sizeof(extension__) - 1; \
5481     if (len < ext_len) { \
5482       is_extension__ = PETSC_FALSE; \
5483     } else { \
5484       PetscCall(PetscStrncmp(filename + len - ext_len, extension__, ext_len, &is_extension__)); \
5485     } \
5486   } while (0)
5487 
5488   CheckExtension(extGmsh, isGmsh);
5489   CheckExtension(extGmsh2, isGmsh2);
5490   CheckExtension(extGmsh4, isGmsh4);
5491   CheckExtension(extCGNS, isCGNS);
5492   CheckExtension(extExodus, isExodus);
5493   if (!isExodus) CheckExtension(extExodus_e, isExodus);
5494   CheckExtension(extGenesis, isGenesis);
5495   CheckExtension(extFluent, isFluent);
5496   CheckExtension(extHDF5, isHDF5);
5497   CheckExtension(extMed, isMed);
5498   CheckExtension(extPLY, isPLY);
5499   CheckExtension(extEGADSLite, isEGADSLite);
5500   CheckExtension(extEGADS, isEGADS);
5501   CheckExtension(extIGES, isIGES);
5502   CheckExtension(extSTEP, isSTEP);
5503   CheckExtension(extCV, isCV);
5504   CheckExtension(extXDMFHDF5, isXDMFHDF5);
5505 
5506 #undef CheckExtension
5507 
5508   if (isGmsh || isGmsh2 || isGmsh4) {
5509     PetscCall(DMPlexCreateGmshFromFile(comm, filename, interpolate, dm));
5510   } else if (isCGNS) {
5511     PetscCall(DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm));
5512   } else if (isExodus || isGenesis) {
5513     PetscCall(DMPlexCreateExodusFromFile(comm, filename, interpolate, dm));
5514   } else if (isFluent) {
5515     PetscCall(DMPlexCreateFluentFromFile(comm, filename, interpolate, dm));
5516   } else if (isHDF5) {
5517     PetscViewer viewer;
5518 
5519     /* PETSC_VIEWER_HDF5_XDMF is used if the filename ends with .xdmf.h5, or if -dm_plex_create_from_hdf5_xdmf option is present */
5520     PetscCall(PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &isXDMFHDF5, NULL));
5521     PetscCall(PetscViewerCreate(comm, &viewer));
5522     PetscCall(PetscViewerSetType(viewer, PETSCVIEWERHDF5));
5523     PetscCall(PetscViewerSetOptionsPrefix(viewer, "dm_plex_create_"));
5524     PetscCall(PetscViewerSetFromOptions(viewer));
5525     PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
5526     PetscCall(PetscViewerFileSetName(viewer, filename));
5527 
5528     PetscCall(DMCreate(comm, dm));
5529     PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5530     PetscCall(DMSetType(*dm, DMPLEX));
5531     if (isXDMFHDF5) PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF));
5532     PetscCall(DMLoad(*dm, viewer));
5533     if (isXDMFHDF5) PetscCall(PetscViewerPopFormat(viewer));
5534     PetscCall(PetscViewerDestroy(&viewer));
5535 
5536     if (interpolate) {
5537       DM idm;
5538 
5539       PetscCall(DMPlexInterpolate(*dm, &idm));
5540       PetscCall(DMDestroy(dm));
5541       *dm = idm;
5542     }
5543   } else if (isMed) {
5544     PetscCall(DMPlexCreateMedFromFile(comm, filename, interpolate, dm));
5545   } else if (isPLY) {
5546     PetscCall(DMPlexCreatePLYFromFile(comm, filename, interpolate, dm));
5547   } else if (isEGADSLite || isEGADS || isIGES || isSTEP) {
5548     if (isEGADSLite) PetscCall(DMPlexCreateEGADSLiteFromFile(comm, filename, dm));
5549     else PetscCall(DMPlexCreateEGADSFromFile(comm, filename, dm));
5550     if (!interpolate) {
5551       DM udm;
5552 
5553       PetscCall(DMPlexUninterpolate(*dm, &udm));
5554       PetscCall(DMDestroy(dm));
5555       *dm = udm;
5556     }
5557   } else if (isCV) {
5558     PetscCall(DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm));
5559   } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
5560   PetscCall(PetscStrlen(plexname, &len));
5561   if (len) PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5562   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5563   PetscFunctionReturn(PETSC_SUCCESS);
5564 }
5565 
5566 /*@C
5567   DMPlexCreateEphemeral - This takes a `DMPlexTransform` and a base `DMPlex` and produces an ephemeral `DM`, meaning one that is created on the fly in response to queries.
5568 
5569   Input Parameters:
5570 + tr     - The `DMPlexTransform`
5571 - prefix - An options prefix, or NULL
5572 
5573   Output Parameter:
5574 . dm - The `DM`
5575 
5576   Level: beginner
5577 
5578   Notes:
5579   An emphemeral mesh is one that is not stored concretely, as in the default `DMPLEX` implementation, but rather is produced on the fly in response to queries, using information from the transform and the base mesh.
5580 
5581 .seealso: `DMPlexCreateFromFile`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
5582 @*/
5583 PetscErrorCode DMPlexCreateEphemeral(DMPlexTransform tr, const char prefix[], DM *dm)
5584 {
5585   DM           bdm, bcdm, cdm;
5586   Vec          coordinates, coordinatesNew;
5587   PetscSection cs;
5588   PetscInt     dim, cdim, Nl;
5589 
5590   PetscFunctionBegin;
5591   PetscCall(DMCreate(PetscObjectComm((PetscObject)tr), dm));
5592   PetscCall(DMSetType(*dm, DMPLEX));
5593   ((DM_Plex *)(*dm)->data)->interpolated = DMPLEX_INTERPOLATED_FULL;
5594   // Handle coordinates
5595   PetscCall(DMPlexTransformGetDM(tr, &bdm));
5596   PetscCall(DMGetCoordinateDim(bdm, &cdim));
5597   PetscCall(DMSetCoordinateDim(*dm, cdim));
5598   PetscCall(DMGetDimension(bdm, &dim));
5599   PetscCall(DMSetDimension(*dm, dim));
5600   PetscCall(DMGetCoordinateDM(bdm, &bcdm));
5601   PetscCall(DMGetCoordinateDM(*dm, &cdm));
5602   PetscCall(DMCopyDisc(bcdm, cdm));
5603   PetscCall(DMGetLocalSection(cdm, &cs));
5604   PetscCall(PetscSectionSetNumFields(cs, 1));
5605   PetscCall(PetscSectionSetFieldComponents(cs, 0, cdim));
5606   PetscCall(DMGetCoordinatesLocal(bdm, &coordinates));
5607   PetscCall(VecDuplicate(coordinates, &coordinatesNew));
5608   PetscCall(VecCopy(coordinates, coordinatesNew));
5609   PetscCall(DMSetCoordinatesLocal(*dm, coordinatesNew));
5610   PetscCall(VecDestroy(&coordinatesNew));
5611 
5612   PetscCall(PetscObjectReference((PetscObject)tr));
5613   PetscCall(DMPlexTransformDestroy(&((DM_Plex *)(*dm)->data)->tr));
5614   ((DM_Plex *)(*dm)->data)->tr = tr;
5615   PetscCall(DMPlexDistributeSetDefault(*dm, PETSC_FALSE));
5616   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)*dm, prefix));
5617   PetscCall(DMSetFromOptions(*dm));
5618 
5619   PetscCall(DMGetNumLabels(bdm, &Nl));
5620   for (PetscInt l = 0; l < Nl; ++l) {
5621     DMLabel     label, labelNew;
5622     const char *lname;
5623     PetscBool   isDepth, isCellType;
5624 
5625     PetscCall(DMGetLabelName(bdm, l, &lname));
5626     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
5627     if (isDepth) continue;
5628     PetscCall(PetscStrcmp(lname, "celltype", &isCellType));
5629     if (isCellType) continue;
5630     PetscCall(DMCreateLabel(*dm, lname));
5631     PetscCall(DMGetLabel(bdm, lname, &label));
5632     PetscCall(DMGetLabel(*dm, lname, &labelNew));
5633     PetscCall(DMLabelSetType(labelNew, DMLABELEPHEMERAL));
5634     PetscCall(DMLabelEphemeralSetLabel(labelNew, label));
5635     PetscCall(DMLabelEphemeralSetTransform(labelNew, tr));
5636     PetscCall(DMLabelSetUp(labelNew));
5637   }
5638   PetscFunctionReturn(PETSC_SUCCESS);
5639 }
5640