xref: /petsc/src/dm/impls/plex/plexcreate.c (revision 73fdd05bb67e49f40fd8fd311695ff6fdf0b9b8a)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h> /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>   /*I   "petscdmplex.h"   I*/
4 #include <petscsf.h>
5 #include <petscdmplextransform.h>
6 #include <petscdmlabelephemeral.h>
7 #include <petsc/private/kernels/blockmatmult.h>
8 #include <petsc/private/kernels/blockinvert.h>
9 
10 PetscLogEvent DMPLEX_CreateFromFile, DMPLEX_BuildFromCellList, DMPLEX_BuildCoordinatesFromCellList;
11 
12 /* External function declarations here */
13 static PetscErrorCode DMInitialize_Plex(DM dm);
14 
15 /* This copies internal things in the Plex structure that we generally want when making a new, related Plex */
16 PetscErrorCode DMPlexCopy_Internal(DM dmin, PetscBool copyPeriodicity, PetscBool copyOverlap, DM dmout)
17 {
18   const PetscReal         *maxCell, *Lstart, *L;
19   PetscBool                dist;
20   DMPlexReorderDefaultFlag reorder;
21 
22   PetscFunctionBegin;
23   if (copyPeriodicity) {
24     PetscCall(DMGetPeriodicity(dmin, &maxCell, &Lstart, &L));
25     PetscCall(DMSetPeriodicity(dmout, maxCell, Lstart, L));
26   }
27   PetscCall(DMPlexDistributeGetDefault(dmin, &dist));
28   PetscCall(DMPlexDistributeSetDefault(dmout, dist));
29   PetscCall(DMPlexReorderGetDefault(dmin, &reorder));
30   PetscCall(DMPlexReorderSetDefault(dmout, reorder));
31   ((DM_Plex *)dmout->data)->useHashLocation = ((DM_Plex *)dmin->data)->useHashLocation;
32   if (copyOverlap) PetscCall(DMPlexSetOverlap_Plex(dmout, dmin, 0));
33   PetscFunctionReturn(PETSC_SUCCESS);
34 }
35 
36 /* Replace dm with the contents of ndm, and then destroy ndm
37    - Share the DM_Plex structure
38    - Share the coordinates
39    - Share the SF
40 */
41 PetscErrorCode DMPlexReplace_Internal(DM dm, DM *ndm)
42 {
43   PetscSF          sf;
44   DM               dmNew = *ndm, coordDM, coarseDM;
45   Vec              coords;
46   const PetscReal *maxCell, *Lstart, *L;
47   PetscInt         dim, cdim;
48 
49   PetscFunctionBegin;
50   if (dm == dmNew) {
51     PetscCall(DMDestroy(ndm));
52     PetscFunctionReturn(PETSC_SUCCESS);
53   }
54   dm->setupcalled = dmNew->setupcalled;
55   PetscCall(DMGetDimension(dmNew, &dim));
56   PetscCall(DMSetDimension(dm, dim));
57   PetscCall(DMGetCoordinateDim(dmNew, &cdim));
58   PetscCall(DMSetCoordinateDim(dm, cdim));
59   PetscCall(DMGetPointSF(dmNew, &sf));
60   PetscCall(DMSetPointSF(dm, sf));
61   PetscCall(DMGetCoordinateDM(dmNew, &coordDM));
62   PetscCall(DMGetCoordinatesLocal(dmNew, &coords));
63   PetscCall(DMSetCoordinateDM(dm, coordDM));
64   PetscCall(DMSetCoordinatesLocal(dm, coords));
65   PetscCall(DMGetCellCoordinateDM(dmNew, &coordDM));
66   PetscCall(DMGetCellCoordinatesLocal(dmNew, &coords));
67   PetscCall(DMSetCellCoordinateDM(dm, coordDM));
68   PetscCall(DMSetCellCoordinatesLocal(dm, coords));
69   /* Do not want to create the coordinate field if it does not already exist, so do not call DMGetCoordinateField() */
70   PetscCall(DMFieldDestroy(&dm->coordinates[0].field));
71   dm->coordinates[0].field            = dmNew->coordinates[0].field;
72   ((DM_Plex *)dmNew->data)->coordFunc = ((DM_Plex *)dm->data)->coordFunc;
73   PetscCall(DMGetPeriodicity(dmNew, &maxCell, &Lstart, &L));
74   PetscCall(DMSetPeriodicity(dm, maxCell, Lstart, L));
75   PetscCall(DMDestroy_Plex(dm));
76   PetscCall(DMInitialize_Plex(dm));
77   dm->data = dmNew->data;
78   ((DM_Plex *)dmNew->data)->refct++;
79   PetscCall(DMPlexGetIsoperiodicFaceSF(dm, &sf));
80   PetscCall(DMPlexSetIsoperiodicFaceSF(dm, sf)); // for the compose function effect on dm
81   PetscCall(DMDestroyLabelLinkList_Internal(dm));
82   PetscCall(DMCopyLabels(dmNew, dm, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL));
83   PetscCall(DMGetCoarseDM(dmNew, &coarseDM));
84   PetscCall(DMSetCoarseDM(dm, coarseDM));
85   PetscCall(DMDestroy(ndm));
86   PetscFunctionReturn(PETSC_SUCCESS);
87 }
88 
89 /* Swap dm with the contents of dmNew
90    - Swap the DM_Plex structure
91    - Swap the coordinates
92    - Swap the point PetscSF
93 */
94 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
95 {
96   DM          coordDMA, coordDMB;
97   Vec         coordsA, coordsB;
98   PetscSF     sfA, sfB;
99   DMField     fieldTmp;
100   void       *tmp;
101   DMLabelLink listTmp;
102   DMLabel     depthTmp;
103   PetscInt    tmpI;
104 
105   PetscFunctionBegin;
106   if (dmA == dmB) PetscFunctionReturn(PETSC_SUCCESS);
107   PetscCall(DMGetPointSF(dmA, &sfA));
108   PetscCall(DMGetPointSF(dmB, &sfB));
109   PetscCall(PetscObjectReference((PetscObject)sfA));
110   PetscCall(DMSetPointSF(dmA, sfB));
111   PetscCall(DMSetPointSF(dmB, sfA));
112   PetscCall(PetscObjectDereference((PetscObject)sfA));
113 
114   PetscCall(DMGetCoordinateDM(dmA, &coordDMA));
115   PetscCall(DMGetCoordinateDM(dmB, &coordDMB));
116   PetscCall(PetscObjectReference((PetscObject)coordDMA));
117   PetscCall(DMSetCoordinateDM(dmA, coordDMB));
118   PetscCall(DMSetCoordinateDM(dmB, coordDMA));
119   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
120 
121   PetscCall(DMGetCoordinatesLocal(dmA, &coordsA));
122   PetscCall(DMGetCoordinatesLocal(dmB, &coordsB));
123   PetscCall(PetscObjectReference((PetscObject)coordsA));
124   PetscCall(DMSetCoordinatesLocal(dmA, coordsB));
125   PetscCall(DMSetCoordinatesLocal(dmB, coordsA));
126   PetscCall(PetscObjectDereference((PetscObject)coordsA));
127 
128   PetscCall(DMGetCellCoordinateDM(dmA, &coordDMA));
129   PetscCall(DMGetCellCoordinateDM(dmB, &coordDMB));
130   PetscCall(PetscObjectReference((PetscObject)coordDMA));
131   PetscCall(DMSetCellCoordinateDM(dmA, coordDMB));
132   PetscCall(DMSetCellCoordinateDM(dmB, coordDMA));
133   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
134 
135   PetscCall(DMGetCellCoordinatesLocal(dmA, &coordsA));
136   PetscCall(DMGetCellCoordinatesLocal(dmB, &coordsB));
137   PetscCall(PetscObjectReference((PetscObject)coordsA));
138   PetscCall(DMSetCellCoordinatesLocal(dmA, coordsB));
139   PetscCall(DMSetCellCoordinatesLocal(dmB, coordsA));
140   PetscCall(PetscObjectDereference((PetscObject)coordsA));
141 
142   fieldTmp                  = dmA->coordinates[0].field;
143   dmA->coordinates[0].field = dmB->coordinates[0].field;
144   dmB->coordinates[0].field = fieldTmp;
145   fieldTmp                  = dmA->coordinates[1].field;
146   dmA->coordinates[1].field = dmB->coordinates[1].field;
147   dmB->coordinates[1].field = fieldTmp;
148   tmp                       = dmA->data;
149   dmA->data                 = dmB->data;
150   dmB->data                 = tmp;
151   listTmp                   = dmA->labels;
152   dmA->labels               = dmB->labels;
153   dmB->labels               = listTmp;
154   depthTmp                  = dmA->depthLabel;
155   dmA->depthLabel           = dmB->depthLabel;
156   dmB->depthLabel           = depthTmp;
157   depthTmp                  = dmA->celltypeLabel;
158   dmA->celltypeLabel        = dmB->celltypeLabel;
159   dmB->celltypeLabel        = depthTmp;
160   tmpI                      = dmA->levelup;
161   dmA->levelup              = dmB->levelup;
162   dmB->levelup              = tmpI;
163   PetscFunctionReturn(PETSC_SUCCESS);
164 }
165 
166 PetscErrorCode DMPlexInterpolateInPlace_Internal(DM dm)
167 {
168   DM idm;
169 
170   PetscFunctionBegin;
171   PetscCall(DMPlexInterpolate(dm, &idm));
172   PetscCall(DMPlexCopyCoordinates(dm, idm));
173   PetscCall(DMPlexReplace_Internal(dm, &idm));
174   PetscFunctionReturn(PETSC_SUCCESS);
175 }
176 
177 /*@C
178   DMPlexCreateCoordinateSpace - Creates a finite element space for the coordinates
179 
180   Collective on dm
181 
182   Input Parameters:
183 + DM        - The DM
184 . degree    - The degree of the finite element or PETSC_DECIDE
185 - coordFunc - An optional function to map new points from refinement to the surface
186 
187   Level: advanced
188 
189 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `PetscPointFunc`, `PetscFECreateLagrange()`, `DMGetCoordinateDM()`
190 @*/
191 PetscErrorCode DMPlexCreateCoordinateSpace(DM dm, PetscInt degree, PetscPointFunc coordFunc)
192 {
193   DM_Plex     *mesh = (DM_Plex *)dm->data;
194   DM           cdm;
195   PetscDS      cds;
196   PetscFE      fe;
197   PetscClassId id;
198 
199   PetscFunctionBegin;
200   PetscCall(DMGetCoordinateDM(dm, &cdm));
201   PetscCall(DMGetDS(cdm, &cds));
202   PetscCall(PetscDSGetDiscretization(cds, 0, (PetscObject *)&fe));
203   PetscCall(PetscObjectGetClassId((PetscObject)fe, &id));
204   if (id != PETSCFE_CLASSID) {
205     PetscInt dim, dE, qorder;
206 
207     PetscCall(DMGetDimension(dm, &dim));
208     PetscCall(DMGetCoordinateDim(dm, &dE));
209     qorder = degree;
210     PetscObjectOptionsBegin((PetscObject)cdm);
211     PetscCall(PetscOptionsBoundedInt("-default_quadrature_order", "Quadrature order is one less than quadrature points per edge", "DMPlexCreateCoordinateSpace", qorder, &qorder, NULL, 0));
212     PetscOptionsEnd();
213     if (degree == PETSC_DECIDE) fe = NULL;
214     else {
215       DMPolytopeType ct;
216       PetscInt       cStart, cEnd, ctTmp;
217 
218       PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
219       // Want to match cell types
220       if (cEnd > cStart) PetscCall(DMPlexGetCellType(dm, cStart, &ct));
221       else ct = DM_POLYTOPE_UNKNOWN;
222       ctTmp = (PetscInt)ct;
223       PetscCallMPI(MPI_Allreduce(MPI_IN_PLACE, &ctTmp, 1, MPIU_INT, MPI_MIN, PetscObjectComm((PetscObject)dm)));
224       ct = (DMPolytopeType)ctTmp;
225       // Work around current bug
226       if (ct == DM_POLYTOPE_SEG_PRISM_TENSOR || ct == DM_POLYTOPE_TRI_PRISM_TENSOR || ct == DM_POLYTOPE_QUAD_PRISM_TENSOR) fe = NULL;
227       else PetscCall(PetscFECreateLagrangeByCell(PETSC_COMM_SELF, dim, dE, ct, degree, qorder, &fe));
228     }
229     if (fe) PetscCall(DMProjectCoordinates(dm, fe));
230     PetscCall(PetscFEDestroy(&fe));
231   }
232   mesh->coordFunc = coordFunc;
233   PetscFunctionReturn(PETSC_SUCCESS);
234 }
235 
236 /*@
237   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
238 
239   Collective
240 
241   Input Parameters:
242 + comm - The communicator for the `DM` object
243 . dim - The spatial dimension
244 . simplex - Flag for simplicial cells, otherwise they are tensor product cells
245 . interpolate - Flag to create intermediate mesh pieces (edges, faces)
246 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
247 
248   Output Parameter:
249 . dm - The `DM` object
250 
251   Level: beginner
252 
253 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMSetType()`, `DMCreate()`
254 @*/
255 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscReal refinementLimit, DM *newdm)
256 {
257   DM          dm;
258   PetscMPIInt rank;
259 
260   PetscFunctionBegin;
261   PetscCall(DMCreate(comm, &dm));
262   PetscCall(DMSetType(dm, DMPLEX));
263   PetscCall(DMSetDimension(dm, dim));
264   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
265   PetscCallMPI(MPI_Comm_rank(comm, &rank));
266   switch (dim) {
267   case 2:
268     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "triangular"));
269     else PetscCall(PetscObjectSetName((PetscObject)dm, "quadrilateral"));
270     break;
271   case 3:
272     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "tetrahedral"));
273     else PetscCall(PetscObjectSetName((PetscObject)dm, "hexahedral"));
274     break;
275   default:
276     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
277   }
278   if (rank) {
279     PetscInt numPoints[2] = {0, 0};
280     PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL));
281   } else {
282     switch (dim) {
283     case 2:
284       if (simplex) {
285         PetscInt    numPoints[2]        = {4, 2};
286         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
287         PetscInt    cones[6]            = {2, 3, 4, 5, 4, 3};
288         PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
289         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
290 
291         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
292       } else {
293         PetscInt    numPoints[2]        = {6, 2};
294         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
295         PetscInt    cones[8]            = {2, 3, 4, 5, 3, 6, 7, 4};
296         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
297         PetscScalar vertexCoords[12]    = {-1.0, -0.5, 0.0, -0.5, 0.0, 0.5, -1.0, 0.5, 1.0, -0.5, 1.0, 0.5};
298 
299         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
300       }
301       break;
302     case 3:
303       if (simplex) {
304         PetscInt    numPoints[2]        = {5, 2};
305         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
306         PetscInt    cones[8]            = {4, 3, 5, 2, 5, 3, 4, 6};
307         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
308         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0};
309 
310         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
311       } else {
312         PetscInt    numPoints[2]         = {12, 2};
313         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
314         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9, 5, 4, 10, 11, 7, 12, 13, 8};
315         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
316         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5, -1.0, 0.5, -0.5, 0.0, 0.5, -0.5, 0.0, -0.5, -0.5, -1.0, -0.5, 0.5, 0.0, -0.5, 0.5, 0.0, 0.5, 0.5, -1.0, 0.5, 0.5, 1.0, 0.5, -0.5, 1.0, -0.5, -0.5, 1.0, -0.5, 0.5, 1.0, 0.5, 0.5};
317 
318         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
319       }
320       break;
321     default:
322       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
323     }
324   }
325   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
326   *newdm = dm;
327   if (refinementLimit > 0.0) {
328     DM          rdm;
329     const char *name;
330 
331     PetscCall(DMPlexSetRefinementUniform(*newdm, PETSC_FALSE));
332     PetscCall(DMPlexSetRefinementLimit(*newdm, refinementLimit));
333     PetscCall(DMRefine(*newdm, comm, &rdm));
334     PetscCall(PetscObjectGetName((PetscObject)*newdm, &name));
335     PetscCall(PetscObjectSetName((PetscObject)rdm, name));
336     PetscCall(DMDestroy(newdm));
337     *newdm = rdm;
338   }
339   if (interpolate) {
340     DM idm;
341 
342     PetscCall(DMPlexInterpolate(*newdm, &idm));
343     PetscCall(DMDestroy(newdm));
344     *newdm = idm;
345   }
346   PetscFunctionReturn(PETSC_SUCCESS);
347 }
348 
349 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
350 {
351   const PetscInt numVertices    = 2;
352   PetscInt       markerRight    = 1;
353   PetscInt       markerLeft     = 1;
354   PetscBool      markerSeparate = PETSC_FALSE;
355   Vec            coordinates;
356   PetscSection   coordSection;
357   PetscScalar   *coords;
358   PetscInt       coordSize;
359   PetscMPIInt    rank;
360   PetscInt       cdim = 1, v;
361 
362   PetscFunctionBegin;
363   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
364   if (markerSeparate) {
365     markerRight = 2;
366     markerLeft  = 1;
367   }
368   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
369   if (rank == 0) {
370     PetscCall(DMPlexSetChart(dm, 0, numVertices));
371     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
372     PetscCall(DMSetLabelValue(dm, "marker", 0, markerLeft));
373     PetscCall(DMSetLabelValue(dm, "marker", 1, markerRight));
374   }
375   PetscCall(DMPlexSymmetrize(dm));
376   PetscCall(DMPlexStratify(dm));
377   /* Build coordinates */
378   PetscCall(DMSetCoordinateDim(dm, cdim));
379   PetscCall(DMGetCoordinateSection(dm, &coordSection));
380   PetscCall(PetscSectionSetNumFields(coordSection, 1));
381   PetscCall(PetscSectionSetChart(coordSection, 0, numVertices));
382   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
383   for (v = 0; v < numVertices; ++v) {
384     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
385     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
386   }
387   PetscCall(PetscSectionSetUp(coordSection));
388   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
389   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
390   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
391   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
392   PetscCall(VecSetBlockSize(coordinates, cdim));
393   PetscCall(VecSetType(coordinates, VECSTANDARD));
394   PetscCall(VecGetArray(coordinates, &coords));
395   coords[0] = lower[0];
396   coords[1] = upper[0];
397   PetscCall(VecRestoreArray(coordinates, &coords));
398   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
399   PetscCall(VecDestroy(&coordinates));
400   PetscFunctionReturn(PETSC_SUCCESS);
401 }
402 
403 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
404 {
405   const PetscInt numVertices    = (edges[0] + 1) * (edges[1] + 1);
406   const PetscInt numEdges       = edges[0] * (edges[1] + 1) + (edges[0] + 1) * edges[1];
407   PetscInt       markerTop      = 1;
408   PetscInt       markerBottom   = 1;
409   PetscInt       markerRight    = 1;
410   PetscInt       markerLeft     = 1;
411   PetscBool      markerSeparate = PETSC_FALSE;
412   Vec            coordinates;
413   PetscSection   coordSection;
414   PetscScalar   *coords;
415   PetscInt       coordSize;
416   PetscMPIInt    rank;
417   PetscInt       v, vx, vy;
418 
419   PetscFunctionBegin;
420   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
421   if (markerSeparate) {
422     markerTop    = 3;
423     markerBottom = 1;
424     markerRight  = 2;
425     markerLeft   = 4;
426   }
427   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
428   if (rank == 0) {
429     PetscInt e, ex, ey;
430 
431     PetscCall(DMPlexSetChart(dm, 0, numEdges + numVertices));
432     for (e = 0; e < numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
433     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
434     for (vx = 0; vx <= edges[0]; vx++) {
435       for (ey = 0; ey < edges[1]; ey++) {
436         PetscInt edge   = vx * edges[1] + ey + edges[0] * (edges[1] + 1);
437         PetscInt vertex = ey * (edges[0] + 1) + vx + numEdges;
438         PetscInt cone[2];
439 
440         cone[0] = vertex;
441         cone[1] = vertex + edges[0] + 1;
442         PetscCall(DMPlexSetCone(dm, edge, cone));
443         if (vx == edges[0]) {
444           PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
445           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
446           if (ey == edges[1] - 1) {
447             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
448             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerRight));
449           }
450         } else if (vx == 0) {
451           PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
452           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
453           if (ey == edges[1] - 1) {
454             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
455             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft));
456           }
457         }
458       }
459     }
460     for (vy = 0; vy <= edges[1]; vy++) {
461       for (ex = 0; ex < edges[0]; ex++) {
462         PetscInt edge   = vy * edges[0] + ex;
463         PetscInt vertex = vy * (edges[0] + 1) + ex + numEdges;
464         PetscInt cone[2];
465 
466         cone[0] = vertex;
467         cone[1] = vertex + 1;
468         PetscCall(DMPlexSetCone(dm, edge, cone));
469         if (vy == edges[1]) {
470           PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
471           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
472           if (ex == edges[0] - 1) {
473             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
474             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerTop));
475           }
476         } else if (vy == 0) {
477           PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
478           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
479           if (ex == edges[0] - 1) {
480             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
481             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom));
482           }
483         }
484       }
485     }
486   }
487   PetscCall(DMPlexSymmetrize(dm));
488   PetscCall(DMPlexStratify(dm));
489   /* Build coordinates */
490   PetscCall(DMSetCoordinateDim(dm, 2));
491   PetscCall(DMGetCoordinateSection(dm, &coordSection));
492   PetscCall(PetscSectionSetNumFields(coordSection, 1));
493   PetscCall(PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices));
494   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 2));
495   for (v = numEdges; v < numEdges + numVertices; ++v) {
496     PetscCall(PetscSectionSetDof(coordSection, v, 2));
497     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 2));
498   }
499   PetscCall(PetscSectionSetUp(coordSection));
500   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
501   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
502   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
503   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
504   PetscCall(VecSetBlockSize(coordinates, 2));
505   PetscCall(VecSetType(coordinates, VECSTANDARD));
506   PetscCall(VecGetArray(coordinates, &coords));
507   for (vy = 0; vy <= edges[1]; ++vy) {
508     for (vx = 0; vx <= edges[0]; ++vx) {
509       coords[(vy * (edges[0] + 1) + vx) * 2 + 0] = lower[0] + ((upper[0] - lower[0]) / edges[0]) * vx;
510       coords[(vy * (edges[0] + 1) + vx) * 2 + 1] = lower[1] + ((upper[1] - lower[1]) / edges[1]) * vy;
511     }
512   }
513   PetscCall(VecRestoreArray(coordinates, &coords));
514   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
515   PetscCall(VecDestroy(&coordinates));
516   PetscFunctionReturn(PETSC_SUCCESS);
517 }
518 
519 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
520 {
521   PetscInt     vertices[3], numVertices;
522   PetscInt     numFaces       = 2 * faces[0] * faces[1] + 2 * faces[1] * faces[2] + 2 * faces[0] * faces[2];
523   PetscInt     markerTop      = 1;
524   PetscInt     markerBottom   = 1;
525   PetscInt     markerFront    = 1;
526   PetscInt     markerBack     = 1;
527   PetscInt     markerRight    = 1;
528   PetscInt     markerLeft     = 1;
529   PetscBool    markerSeparate = PETSC_FALSE;
530   Vec          coordinates;
531   PetscSection coordSection;
532   PetscScalar *coords;
533   PetscInt     coordSize;
534   PetscMPIInt  rank;
535   PetscInt     v, vx, vy, vz;
536   PetscInt     voffset, iface = 0, cone[4];
537 
538   PetscFunctionBegin;
539   PetscCheck(faces[0] >= 1 && faces[1] >= 1 && faces[2] >= 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
540   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
541   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
542   if (markerSeparate) {
543     markerBottom = 1;
544     markerTop    = 2;
545     markerFront  = 3;
546     markerBack   = 4;
547     markerRight  = 5;
548     markerLeft   = 6;
549   }
550   vertices[0] = faces[0] + 1;
551   vertices[1] = faces[1] + 1;
552   vertices[2] = faces[2] + 1;
553   numVertices = vertices[0] * vertices[1] * vertices[2];
554   if (rank == 0) {
555     PetscInt f;
556 
557     PetscCall(DMPlexSetChart(dm, 0, numFaces + numVertices));
558     for (f = 0; f < numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
559     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
560 
561     /* Side 0 (Top) */
562     for (vy = 0; vy < faces[1]; vy++) {
563       for (vx = 0; vx < faces[0]; vx++) {
564         voffset = numFaces + vertices[0] * vertices[1] * (vertices[2] - 1) + vy * vertices[0] + vx;
565         cone[0] = voffset;
566         cone[1] = voffset + 1;
567         cone[2] = voffset + vertices[0] + 1;
568         cone[3] = voffset + vertices[0];
569         PetscCall(DMPlexSetCone(dm, iface, cone));
570         PetscCall(DMSetLabelValue(dm, "marker", iface, markerTop));
571         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerTop));
572         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerTop));
573         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerTop));
574         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerTop));
575         iface++;
576       }
577     }
578 
579     /* Side 1 (Bottom) */
580     for (vy = 0; vy < faces[1]; vy++) {
581       for (vx = 0; vx < faces[0]; vx++) {
582         voffset = numFaces + vy * (faces[0] + 1) + vx;
583         cone[0] = voffset + 1;
584         cone[1] = voffset;
585         cone[2] = voffset + vertices[0];
586         cone[3] = voffset + vertices[0] + 1;
587         PetscCall(DMPlexSetCone(dm, iface, cone));
588         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBottom));
589         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBottom));
590         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBottom));
591         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerBottom));
592         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerBottom));
593         iface++;
594       }
595     }
596 
597     /* Side 2 (Front) */
598     for (vz = 0; vz < faces[2]; vz++) {
599       for (vx = 0; vx < faces[0]; vx++) {
600         voffset = numFaces + vz * vertices[0] * vertices[1] + vx;
601         cone[0] = voffset;
602         cone[1] = voffset + 1;
603         cone[2] = voffset + vertices[0] * vertices[1] + 1;
604         cone[3] = voffset + vertices[0] * vertices[1];
605         PetscCall(DMPlexSetCone(dm, iface, cone));
606         PetscCall(DMSetLabelValue(dm, "marker", iface, markerFront));
607         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerFront));
608         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerFront));
609         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerFront));
610         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerFront));
611         iface++;
612       }
613     }
614 
615     /* Side 3 (Back) */
616     for (vz = 0; vz < faces[2]; vz++) {
617       for (vx = 0; vx < faces[0]; vx++) {
618         voffset = numFaces + vz * vertices[0] * vertices[1] + vertices[0] * (vertices[1] - 1) + vx;
619         cone[0] = voffset + vertices[0] * vertices[1];
620         cone[1] = voffset + vertices[0] * vertices[1] + 1;
621         cone[2] = voffset + 1;
622         cone[3] = voffset;
623         PetscCall(DMPlexSetCone(dm, iface, cone));
624         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBack));
625         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBack));
626         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBack));
627         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerBack));
628         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerBack));
629         iface++;
630       }
631     }
632 
633     /* Side 4 (Left) */
634     for (vz = 0; vz < faces[2]; vz++) {
635       for (vy = 0; vy < faces[1]; vy++) {
636         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0];
637         cone[0] = voffset;
638         cone[1] = voffset + vertices[0] * vertices[1];
639         cone[2] = voffset + vertices[0] * vertices[1] + vertices[0];
640         cone[3] = voffset + vertices[0];
641         PetscCall(DMPlexSetCone(dm, iface, cone));
642         PetscCall(DMSetLabelValue(dm, "marker", iface, markerLeft));
643         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerLeft));
644         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerLeft));
645         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[1] + 0, markerLeft));
646         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerLeft));
647         iface++;
648       }
649     }
650 
651     /* Side 5 (Right) */
652     for (vz = 0; vz < faces[2]; vz++) {
653       for (vy = 0; vy < faces[1]; vy++) {
654         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0] + faces[0];
655         cone[0] = voffset + vertices[0] * vertices[1];
656         cone[1] = voffset;
657         cone[2] = voffset + vertices[0];
658         cone[3] = voffset + vertices[0] * vertices[1] + vertices[0];
659         PetscCall(DMPlexSetCone(dm, iface, cone));
660         PetscCall(DMSetLabelValue(dm, "marker", iface, markerRight));
661         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerRight));
662         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerRight));
663         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerRight));
664         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerRight));
665         iface++;
666       }
667     }
668   }
669   PetscCall(DMPlexSymmetrize(dm));
670   PetscCall(DMPlexStratify(dm));
671   /* Build coordinates */
672   PetscCall(DMSetCoordinateDim(dm, 3));
673   PetscCall(DMGetCoordinateSection(dm, &coordSection));
674   PetscCall(PetscSectionSetNumFields(coordSection, 1));
675   PetscCall(PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices));
676   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 3));
677   for (v = numFaces; v < numFaces + numVertices; ++v) {
678     PetscCall(PetscSectionSetDof(coordSection, v, 3));
679     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 3));
680   }
681   PetscCall(PetscSectionSetUp(coordSection));
682   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
683   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
684   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
685   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
686   PetscCall(VecSetBlockSize(coordinates, 3));
687   PetscCall(VecSetType(coordinates, VECSTANDARD));
688   PetscCall(VecGetArray(coordinates, &coords));
689   for (vz = 0; vz <= faces[2]; ++vz) {
690     for (vy = 0; vy <= faces[1]; ++vy) {
691       for (vx = 0; vx <= faces[0]; ++vx) {
692         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 0] = lower[0] + ((upper[0] - lower[0]) / faces[0]) * vx;
693         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 1] = lower[1] + ((upper[1] - lower[1]) / faces[1]) * vy;
694         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 2] = lower[2] + ((upper[2] - lower[2]) / faces[2]) * vz;
695       }
696     }
697   }
698   PetscCall(VecRestoreArray(coordinates, &coords));
699   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
700   PetscCall(VecDestroy(&coordinates));
701   PetscFunctionReturn(PETSC_SUCCESS);
702 }
703 
704 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate)
705 {
706   PetscFunctionBegin;
707   PetscValidLogicalCollectiveInt(dm, dim, 2);
708   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
709   PetscCall(DMSetDimension(dm, dim - 1));
710   PetscCall(DMSetCoordinateDim(dm, dim));
711   switch (dim) {
712   case 1:
713     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(dm, lower, upper, faces));
714     break;
715   case 2:
716     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(dm, lower, upper, faces));
717     break;
718   case 3:
719     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(dm, lower, upper, faces));
720     break;
721   default:
722     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Dimension not supported: %" PetscInt_FMT, dim);
723   }
724   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
725   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
726   PetscFunctionReturn(PETSC_SUCCESS);
727 }
728 
729 /*@C
730   DMPlexCreateBoxSurfaceMesh - Creates a mesh on the surface of the tensor product of unit intervals (box) using tensor cells (hexahedra).
731 
732   Collective
733 
734   Input Parameters:
735 + comm        - The communicator for the `DM` object
736 . dim         - The spatial dimension of the box, so the resulting mesh is has dimension dim-1
737 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
738 . lower       - The lower left corner, or NULL for (0, 0, 0)
739 . upper       - The upper right corner, or NULL for (1, 1, 1)
740 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
741 
742   Output Parameter:
743 . dm  - The `DM` object
744 
745   Level: beginner
746 
747 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateBoxMesh()`, `DMPlexCreateFromFile()`, `DMSetType()`, `DMCreate()`
748 @*/
749 PetscErrorCode DMPlexCreateBoxSurfaceMesh(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate, DM *dm)
750 {
751   PetscInt  fac[3] = {1, 1, 1};
752   PetscReal low[3] = {0, 0, 0};
753   PetscReal upp[3] = {1, 1, 1};
754 
755   PetscFunctionBegin;
756   PetscCall(DMCreate(comm, dm));
757   PetscCall(DMSetType(*dm, DMPLEX));
758   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(*dm, dim, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, interpolate));
759   PetscFunctionReturn(PETSC_SUCCESS);
760 }
761 
762 static PetscErrorCode DMPlexCreateLineMesh_Internal(DM dm, PetscInt segments, PetscReal lower, PetscReal upper, DMBoundaryType bd)
763 {
764   PetscInt     i, fStart, fEnd, numCells = 0, numVerts = 0;
765   PetscInt     numPoints[2], *coneSize, *cones, *coneOrientations;
766   PetscScalar *vertexCoords;
767   PetscReal    L, maxCell;
768   PetscBool    markerSeparate = PETSC_FALSE;
769   PetscInt     markerLeft = 1, faceMarkerLeft = 1;
770   PetscInt     markerRight = 1, faceMarkerRight = 2;
771   PetscBool    wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
772   PetscMPIInt  rank;
773 
774   PetscFunctionBegin;
775   PetscValidPointer(dm, 1);
776 
777   PetscCall(DMSetDimension(dm, 1));
778   PetscCall(DMCreateLabel(dm, "marker"));
779   PetscCall(DMCreateLabel(dm, "Face Sets"));
780 
781   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
782   if (rank == 0) numCells = segments;
783   if (rank == 0) numVerts = segments + (wrap ? 0 : 1);
784 
785   numPoints[0] = numVerts;
786   numPoints[1] = numCells;
787   PetscCall(PetscMalloc4(numCells + numVerts, &coneSize, numCells * 2, &cones, numCells + numVerts, &coneOrientations, numVerts, &vertexCoords));
788   PetscCall(PetscArrayzero(coneOrientations, numCells + numVerts));
789   for (i = 0; i < numCells; ++i) coneSize[i] = 2;
790   for (i = 0; i < numVerts; ++i) coneSize[numCells + i] = 0;
791   for (i = 0; i < numCells; ++i) {
792     cones[2 * i]     = numCells + i % numVerts;
793     cones[2 * i + 1] = numCells + (i + 1) % numVerts;
794   }
795   for (i = 0; i < numVerts; ++i) vertexCoords[i] = lower + (upper - lower) * ((PetscReal)i / (PetscReal)numCells);
796   PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
797   PetscCall(PetscFree4(coneSize, cones, coneOrientations, vertexCoords));
798 
799   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
800   if (markerSeparate) {
801     markerLeft  = faceMarkerLeft;
802     markerRight = faceMarkerRight;
803   }
804   if (!wrap && rank == 0) {
805     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
806     PetscCall(DMSetLabelValue(dm, "marker", fStart, markerLeft));
807     PetscCall(DMSetLabelValue(dm, "marker", fEnd - 1, markerRight));
808     PetscCall(DMSetLabelValue(dm, "Face Sets", fStart, faceMarkerLeft));
809     PetscCall(DMSetLabelValue(dm, "Face Sets", fEnd - 1, faceMarkerRight));
810   }
811   if (wrap) {
812     L       = upper - lower;
813     maxCell = (PetscReal)1.1 * (L / (PetscReal)PetscMax(1, segments));
814     PetscCall(DMSetPeriodicity(dm, &maxCell, &lower, &L));
815   }
816   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
817   PetscFunctionReturn(PETSC_SUCCESS);
818 }
819 
820 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
821 {
822   DM      boundary, vol;
823   DMLabel bdlabel;
824 
825   PetscFunctionBegin;
826   PetscValidPointer(dm, 1);
827   for (PetscInt i = 0; i < dim; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
828   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &boundary));
829   PetscCall(DMSetType(boundary, DMPLEX));
830   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(boundary, dim, faces, lower, upper, PETSC_FALSE));
831   PetscCall(DMPlexGenerate(boundary, NULL, interpolate, &vol));
832   PetscCall(DMGetLabel(vol, "marker", &bdlabel));
833   if (bdlabel) PetscCall(DMPlexLabelComplete(vol, bdlabel));
834   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_FALSE, vol));
835   PetscCall(DMPlexReplace_Internal(dm, &vol));
836   PetscCall(DMDestroy(&boundary));
837   PetscFunctionReturn(PETSC_SUCCESS);
838 }
839 
840 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
841 {
842   DMLabel     cutLabel  = NULL;
843   PetscInt    markerTop = 1, faceMarkerTop = 1;
844   PetscInt    markerBottom = 1, faceMarkerBottom = 1;
845   PetscInt    markerFront = 1, faceMarkerFront = 1;
846   PetscInt    markerBack = 1, faceMarkerBack = 1;
847   PetscInt    markerRight = 1, faceMarkerRight = 1;
848   PetscInt    markerLeft = 1, faceMarkerLeft = 1;
849   PetscInt    dim;
850   PetscBool   markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
851   PetscMPIInt rank;
852 
853   PetscFunctionBegin;
854   PetscCall(DMGetDimension(dm, &dim));
855   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
856   PetscCall(DMCreateLabel(dm, "marker"));
857   PetscCall(DMCreateLabel(dm, "Face Sets"));
858   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
859   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST || bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST || bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
860     if (cutMarker) {
861       PetscCall(DMCreateLabel(dm, "periodic_cut"));
862       PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
863     }
864   }
865   switch (dim) {
866   case 2:
867     faceMarkerTop    = 3;
868     faceMarkerBottom = 1;
869     faceMarkerRight  = 2;
870     faceMarkerLeft   = 4;
871     break;
872   case 3:
873     faceMarkerBottom = 1;
874     faceMarkerTop    = 2;
875     faceMarkerFront  = 3;
876     faceMarkerBack   = 4;
877     faceMarkerRight  = 5;
878     faceMarkerLeft   = 6;
879     break;
880   default:
881     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
882   }
883   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
884   if (markerSeparate) {
885     markerBottom = faceMarkerBottom;
886     markerTop    = faceMarkerTop;
887     markerFront  = faceMarkerFront;
888     markerBack   = faceMarkerBack;
889     markerRight  = faceMarkerRight;
890     markerLeft   = faceMarkerLeft;
891   }
892   {
893     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
894     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
895     const PetscInt numZEdges    = rank == 0 ? edges[2] : 0;
896     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0] + 1) : 0;
897     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1] + 1) : 0;
898     const PetscInt numZVertices = rank == 0 ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2] + 1) : 0;
899     const PetscInt numCells     = numXEdges * numYEdges * numZEdges;
900     const PetscInt numXFaces    = numYEdges * numZEdges;
901     const PetscInt numYFaces    = numXEdges * numZEdges;
902     const PetscInt numZFaces    = numXEdges * numYEdges;
903     const PetscInt numTotXFaces = numXVertices * numXFaces;
904     const PetscInt numTotYFaces = numYVertices * numYFaces;
905     const PetscInt numTotZFaces = numZVertices * numZFaces;
906     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
907     const PetscInt numTotXEdges = numXEdges * numYVertices * numZVertices;
908     const PetscInt numTotYEdges = numYEdges * numXVertices * numZVertices;
909     const PetscInt numTotZEdges = numZEdges * numXVertices * numYVertices;
910     const PetscInt numVertices  = numXVertices * numYVertices * numZVertices;
911     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
912     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
913     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
914     const PetscInt firstYFace   = firstXFace + numTotXFaces;
915     const PetscInt firstZFace   = firstYFace + numTotYFaces;
916     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
917     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
918     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
919     Vec            coordinates;
920     PetscSection   coordSection;
921     PetscScalar   *coords;
922     PetscInt       coordSize;
923     PetscInt       v, vx, vy, vz;
924     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
925 
926     PetscCall(DMPlexSetChart(dm, 0, numCells + numFaces + numEdges + numVertices));
927     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
928     for (f = firstXFace; f < firstXFace + numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
929     for (e = firstXEdge; e < firstXEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
930     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
931     /* Build cells */
932     for (fz = 0; fz < numZEdges; ++fz) {
933       for (fy = 0; fy < numYEdges; ++fy) {
934         for (fx = 0; fx < numXEdges; ++fx) {
935           PetscInt cell  = (fz * numYEdges + fy) * numXEdges + fx;
936           PetscInt faceB = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
937           PetscInt faceT = firstZFace + (fy * numXEdges + fx) * numZVertices + ((fz + 1) % numZVertices);
938           PetscInt faceF = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
939           PetscInt faceK = firstYFace + (fz * numXEdges + fx) * numYVertices + ((fy + 1) % numYVertices);
940           PetscInt faceL = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
941           PetscInt faceR = firstXFace + (fz * numYEdges + fy) * numXVertices + ((fx + 1) % numXVertices);
942           /* B,  T,  F,  K,  R,  L */
943           PetscInt ornt[6] = {-2, 0, 0, -3, 0, -2}; /* ??? */
944           PetscInt cone[6];
945 
946           /* no boundary twisting in 3D */
947           cone[0] = faceB;
948           cone[1] = faceT;
949           cone[2] = faceF;
950           cone[3] = faceK;
951           cone[4] = faceR;
952           cone[5] = faceL;
953           PetscCall(DMPlexSetCone(dm, cell, cone));
954           PetscCall(DMPlexSetConeOrientation(dm, cell, ornt));
955           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
956           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
957           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
958         }
959       }
960     }
961     /* Build x faces */
962     for (fz = 0; fz < numZEdges; ++fz) {
963       for (fy = 0; fy < numYEdges; ++fy) {
964         for (fx = 0; fx < numXVertices; ++fx) {
965           PetscInt face    = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
966           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
967           PetscInt edgeR   = firstZEdge + (((fy + 1) % numYVertices) * numXVertices + fx) * numZEdges + fz;
968           PetscInt edgeB   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
969           PetscInt edgeT   = firstYEdge + (((fz + 1) % numZVertices) * numXVertices + fx) * numYEdges + fy;
970           PetscInt ornt[4] = {0, 0, -1, -1};
971           PetscInt cone[4];
972 
973           if (dim == 3) {
974             /* markers */
975             if (bdX != DM_BOUNDARY_PERIODIC) {
976               if (fx == numXVertices - 1) {
977                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight));
978                 PetscCall(DMSetLabelValue(dm, "marker", face, markerRight));
979               } else if (fx == 0) {
980                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft));
981                 PetscCall(DMSetLabelValue(dm, "marker", face, markerLeft));
982               }
983             }
984           }
985           cone[0] = edgeB;
986           cone[1] = edgeR;
987           cone[2] = edgeT;
988           cone[3] = edgeL;
989           PetscCall(DMPlexSetCone(dm, face, cone));
990           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
991         }
992       }
993     }
994     /* Build y faces */
995     for (fz = 0; fz < numZEdges; ++fz) {
996       for (fx = 0; fx < numXEdges; ++fx) {
997         for (fy = 0; fy < numYVertices; ++fy) {
998           PetscInt face    = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
999           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
1000           PetscInt edgeR   = firstZEdge + (fy * numXVertices + ((fx + 1) % numXVertices)) * numZEdges + fz;
1001           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1002           PetscInt edgeT   = firstXEdge + (((fz + 1) % numZVertices) * numYVertices + fy) * numXEdges + fx;
1003           PetscInt ornt[4] = {0, 0, -1, -1};
1004           PetscInt cone[4];
1005 
1006           if (dim == 3) {
1007             /* markers */
1008             if (bdY != DM_BOUNDARY_PERIODIC) {
1009               if (fy == numYVertices - 1) {
1010                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack));
1011                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBack));
1012               } else if (fy == 0) {
1013                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront));
1014                 PetscCall(DMSetLabelValue(dm, "marker", face, markerFront));
1015               }
1016             }
1017           }
1018           cone[0] = edgeB;
1019           cone[1] = edgeR;
1020           cone[2] = edgeT;
1021           cone[3] = edgeL;
1022           PetscCall(DMPlexSetCone(dm, face, cone));
1023           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1024         }
1025       }
1026     }
1027     /* Build z faces */
1028     for (fy = 0; fy < numYEdges; ++fy) {
1029       for (fx = 0; fx < numXEdges; ++fx) {
1030         for (fz = 0; fz < numZVertices; fz++) {
1031           PetscInt face    = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
1032           PetscInt edgeL   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
1033           PetscInt edgeR   = firstYEdge + (fz * numXVertices + ((fx + 1) % numXVertices)) * numYEdges + fy;
1034           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1035           PetscInt edgeT   = firstXEdge + (fz * numYVertices + ((fy + 1) % numYVertices)) * numXEdges + fx;
1036           PetscInt ornt[4] = {0, 0, -1, -1};
1037           PetscInt cone[4];
1038 
1039           if (dim == 2) {
1040             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges - 1) {
1041               edgeR += numYEdges - 1 - 2 * fy;
1042               ornt[1] = -1;
1043             }
1044             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges - 1) {
1045               edgeT += numXEdges - 1 - 2 * fx;
1046               ornt[2] = 0;
1047             }
1048             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1049             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1050           } else {
1051             /* markers */
1052             if (bdZ != DM_BOUNDARY_PERIODIC) {
1053               if (fz == numZVertices - 1) {
1054                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop));
1055                 PetscCall(DMSetLabelValue(dm, "marker", face, markerTop));
1056               } else if (fz == 0) {
1057                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom));
1058                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBottom));
1059               }
1060             }
1061           }
1062           cone[0] = edgeB;
1063           cone[1] = edgeR;
1064           cone[2] = edgeT;
1065           cone[3] = edgeL;
1066           PetscCall(DMPlexSetCone(dm, face, cone));
1067           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1068         }
1069       }
1070     }
1071     /* Build Z edges*/
1072     for (vy = 0; vy < numYVertices; vy++) {
1073       for (vx = 0; vx < numXVertices; vx++) {
1074         for (ez = 0; ez < numZEdges; ez++) {
1075           const PetscInt edge    = firstZEdge + (vy * numXVertices + vx) * numZEdges + ez;
1076           const PetscInt vertexB = firstVertex + (ez * numYVertices + vy) * numXVertices + vx;
1077           const PetscInt vertexT = firstVertex + (((ez + 1) % numZVertices) * numYVertices + vy) * numXVertices + vx;
1078           PetscInt       cone[2];
1079 
1080           cone[0] = vertexB;
1081           cone[1] = vertexT;
1082           PetscCall(DMPlexSetCone(dm, edge, cone));
1083           if (dim == 3) {
1084             if (bdX != DM_BOUNDARY_PERIODIC) {
1085               if (vx == numXVertices - 1) {
1086                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1087                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1088                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1089               } else if (vx == 0) {
1090                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1091                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1092                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1093               }
1094             }
1095             if (bdY != DM_BOUNDARY_PERIODIC) {
1096               if (vy == numYVertices - 1) {
1097                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1098                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1099                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1100               } else if (vy == 0) {
1101                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1102                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1103                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1104               }
1105             }
1106           }
1107         }
1108       }
1109     }
1110     /* Build Y edges*/
1111     for (vz = 0; vz < numZVertices; vz++) {
1112       for (vx = 0; vx < numXVertices; vx++) {
1113         for (ey = 0; ey < numYEdges; ey++) {
1114           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges - 1) ? (numXVertices - vx - 1) : (vz * numYVertices + ((ey + 1) % numYVertices)) * numXVertices + vx;
1115           const PetscInt edge    = firstYEdge + (vz * numXVertices + vx) * numYEdges + ey;
1116           const PetscInt vertexF = firstVertex + (vz * numYVertices + ey) * numXVertices + vx;
1117           const PetscInt vertexK = firstVertex + nextv;
1118           PetscInt       cone[2];
1119 
1120           cone[0] = vertexF;
1121           cone[1] = vertexK;
1122           PetscCall(DMPlexSetCone(dm, edge, cone));
1123           if (dim == 2) {
1124             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1125               if (vx == numXVertices - 1) {
1126                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight));
1127                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1128                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1129                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1130               } else if (vx == 0) {
1131                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft));
1132                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1133                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1134                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1135               }
1136             } else {
1137               if (vx == 0 && cutLabel) {
1138                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1139                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1140                 if (ey == numYEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1141               }
1142             }
1143           } else {
1144             if (bdX != DM_BOUNDARY_PERIODIC) {
1145               if (vx == numXVertices - 1) {
1146                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1147                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1148                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1149               } else if (vx == 0) {
1150                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1151                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1152                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1153               }
1154             }
1155             if (bdZ != DM_BOUNDARY_PERIODIC) {
1156               if (vz == numZVertices - 1) {
1157                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1158                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1159                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1160               } else if (vz == 0) {
1161                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1162                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1163                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1164               }
1165             }
1166           }
1167         }
1168       }
1169     }
1170     /* Build X edges*/
1171     for (vz = 0; vz < numZVertices; vz++) {
1172       for (vy = 0; vy < numYVertices; vy++) {
1173         for (ex = 0; ex < numXEdges; ex++) {
1174           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges - 1) ? (numYVertices - vy - 1) * numXVertices : (vz * numYVertices + vy) * numXVertices + (ex + 1) % numXVertices;
1175           const PetscInt edge    = firstXEdge + (vz * numYVertices + vy) * numXEdges + ex;
1176           const PetscInt vertexL = firstVertex + (vz * numYVertices + vy) * numXVertices + ex;
1177           const PetscInt vertexR = firstVertex + nextv;
1178           PetscInt       cone[2];
1179 
1180           cone[0] = vertexL;
1181           cone[1] = vertexR;
1182           PetscCall(DMPlexSetCone(dm, edge, cone));
1183           if (dim == 2) {
1184             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1185               if (vy == numYVertices - 1) {
1186                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop));
1187                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1188                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1189                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1190               } else if (vy == 0) {
1191                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom));
1192                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1193                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1194                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1195               }
1196             } else {
1197               if (vy == 0 && cutLabel) {
1198                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1199                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1200                 if (ex == numXEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1201               }
1202             }
1203           } else {
1204             if (bdY != DM_BOUNDARY_PERIODIC) {
1205               if (vy == numYVertices - 1) {
1206                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1207                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1208                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1209               } else if (vy == 0) {
1210                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1211                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1212                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1213               }
1214             }
1215             if (bdZ != DM_BOUNDARY_PERIODIC) {
1216               if (vz == numZVertices - 1) {
1217                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1218                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1219                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1220               } else if (vz == 0) {
1221                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1222                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1223                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1224               }
1225             }
1226           }
1227         }
1228       }
1229     }
1230     PetscCall(DMPlexSymmetrize(dm));
1231     PetscCall(DMPlexStratify(dm));
1232     /* Build coordinates */
1233     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1234     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1235     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1236     PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVertices));
1237     for (v = firstVertex; v < firstVertex + numVertices; ++v) {
1238       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1239       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1240     }
1241     PetscCall(PetscSectionSetUp(coordSection));
1242     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1243     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1244     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1245     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1246     PetscCall(VecSetBlockSize(coordinates, dim));
1247     PetscCall(VecSetType(coordinates, VECSTANDARD));
1248     PetscCall(VecGetArray(coordinates, &coords));
1249     for (vz = 0; vz < numZVertices; ++vz) {
1250       for (vy = 0; vy < numYVertices; ++vy) {
1251         for (vx = 0; vx < numXVertices; ++vx) {
1252           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * vx;
1253           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * vy;
1254           if (dim == 3) coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 2] = lower[2] + ((upper[2] - lower[2]) / numZEdges) * vz;
1255         }
1256       }
1257     }
1258     PetscCall(VecRestoreArray(coordinates, &coords));
1259     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1260     PetscCall(VecDestroy(&coordinates));
1261   }
1262   PetscFunctionReturn(PETSC_SUCCESS);
1263 }
1264 
1265 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1266 {
1267   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1268   PetscInt       fac[3] = {0, 0, 0}, d;
1269 
1270   PetscFunctionBegin;
1271   PetscValidPointer(dm, 1);
1272   PetscValidLogicalCollectiveInt(dm, dim, 2);
1273   PetscCall(DMSetDimension(dm, dim));
1274   for (d = 0; d < dim; ++d) {
1275     fac[d] = faces[d];
1276     bdt[d] = periodicity[d];
1277   }
1278   PetscCall(DMPlexCreateCubeMesh_Internal(dm, lower, upper, fac, bdt[0], bdt[1], bdt[2]));
1279   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST || periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST || (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
1280     PetscReal L[3]       = {-1., -1., 0.};
1281     PetscReal maxCell[3] = {-1., -1., 0.};
1282 
1283     for (d = 0; d < dim; ++d) {
1284       if (periodicity[d] != DM_BOUNDARY_NONE) {
1285         L[d]       = upper[d] - lower[d];
1286         maxCell[d] = 1.1 * (L[d] / PetscMax(1, faces[d]));
1287       }
1288     }
1289     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1290   }
1291   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
1292   PetscFunctionReturn(PETSC_SUCCESS);
1293 }
1294 
1295 static PetscErrorCode DMPlexCreateBoxMesh_Internal(DM dm, DMPlexShape shape, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1296 {
1297   PetscFunctionBegin;
1298   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
1299   if (shape == DM_SHAPE_ZBOX) PetscCall(DMPlexCreateBoxMesh_Tensor_SFC_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1300   else if (dim == 1) PetscCall(DMPlexCreateLineMesh_Internal(dm, faces[0], lower[0], upper[0], periodicity[0]));
1301   else if (simplex) PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1302   else PetscCall(DMPlexCreateBoxMesh_Tensor_Internal(dm, dim, faces, lower, upper, periodicity));
1303   if (!interpolate && dim > 1 && !simplex) {
1304     DM udm;
1305 
1306     PetscCall(DMPlexUninterpolate(dm, &udm));
1307     PetscCall(DMPlexCopyCoordinates(dm, udm));
1308     PetscCall(DMPlexReplace_Internal(dm, &udm));
1309   }
1310   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
1311   PetscFunctionReturn(PETSC_SUCCESS);
1312 }
1313 
1314 /*@C
1315   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
1316 
1317   Collective
1318 
1319   Input Parameters:
1320 + comm        - The communicator for the `DM` object
1321 . dim         - The spatial dimension
1322 . simplex     - `PETSC_TRUE` for simplices, `PETSC_FALSE` for tensor cells
1323 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1324 . lower       - The lower left corner, or NULL for (0, 0, 0)
1325 . upper       - The upper right corner, or NULL for (1, 1, 1)
1326 . periodicity - The boundary type for the X,Y,Z direction, or NULL for `DM_BOUNDARY_NONE`
1327 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1328 
1329   Output Parameter:
1330 . dm  - The `DM` object
1331 
1332   Level: beginner
1333 
1334   Note:
1335    To customize this mesh using options, use
1336 .vb
1337   DMCreate(comm, &dm);
1338   DMSetType(dm, DMPLEX);
1339   DMSetFromOptions(dm);
1340 .ve
1341 and use the options in `DMSetFromOptions()`.
1342 
1343   Here is the numbering returned for 2 faces in each direction for tensor cells:
1344 .vb
1345  10---17---11---18----12
1346   |         |         |
1347   |         |         |
1348  20    2   22    3    24
1349   |         |         |
1350   |         |         |
1351   7---15----8---16----9
1352   |         |         |
1353   |         |         |
1354  19    0   21    1   23
1355   |         |         |
1356   |         |         |
1357   4---13----5---14----6
1358 .ve
1359 and for simplicial cells
1360 .vb
1361  14----8---15----9----16
1362   |\     5  |\      7 |
1363   | \       | \       |
1364  13   2    14    3    15
1365   | 4   \   | 6   \   |
1366   |       \ |       \ |
1367  11----6---12----7----13
1368   |\        |\        |
1369   | \    1  | \     3 |
1370  10   0    11    1    12
1371   | 0   \   | 2   \   |
1372   |       \ |       \ |
1373   8----4----9----5----10
1374 .ve
1375 
1376 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
1377 @*/
1378 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
1379 {
1380   PetscInt       fac[3] = {1, 1, 1};
1381   PetscReal      low[3] = {0, 0, 0};
1382   PetscReal      upp[3] = {1, 1, 1};
1383   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1384 
1385   PetscFunctionBegin;
1386   PetscCall(DMCreate(comm, dm));
1387   PetscCall(DMSetType(*dm, DMPLEX));
1388   PetscCall(DMPlexCreateBoxMesh_Internal(*dm, DM_SHAPE_BOX, dim, simplex, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt, interpolate));
1389   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1390   PetscFunctionReturn(PETSC_SUCCESS);
1391 }
1392 
1393 static PetscErrorCode DMPlexCreateWedgeBoxMesh_Internal(DM dm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1394 {
1395   DM       bdm, vol;
1396   PetscInt i;
1397 
1398   PetscFunctionBegin;
1399   for (i = 0; i < 3; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity not yet supported");
1400   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &bdm));
1401   PetscCall(DMSetType(bdm, DMPLEX));
1402   PetscCall(DMSetDimension(bdm, 2));
1403   PetscCall(PetscLogEventBegin(DMPLEX_Generate, bdm, 0, 0, 0));
1404   PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(bdm, 2, faces, lower, upper, periodicity, PETSC_TRUE));
1405   PetscCall(DMPlexExtrude(bdm, faces[2], upper[2] - lower[2], PETSC_TRUE, PETSC_FALSE, NULL, NULL, &vol));
1406   PetscCall(PetscLogEventEnd(DMPLEX_Generate, bdm, 0, 0, 0));
1407   PetscCall(DMDestroy(&bdm));
1408   PetscCall(DMPlexReplace_Internal(dm, &vol));
1409   if (lower[2] != 0.0) {
1410     Vec          v;
1411     PetscScalar *x;
1412     PetscInt     cDim, n;
1413 
1414     PetscCall(DMGetCoordinatesLocal(dm, &v));
1415     PetscCall(VecGetBlockSize(v, &cDim));
1416     PetscCall(VecGetLocalSize(v, &n));
1417     PetscCall(VecGetArray(v, &x));
1418     x += cDim;
1419     for (i = 0; i < n; i += cDim) x[i] += lower[2];
1420     PetscCall(VecRestoreArray(v, &x));
1421     PetscCall(DMSetCoordinatesLocal(dm, v));
1422   }
1423   PetscFunctionReturn(PETSC_SUCCESS);
1424 }
1425 
1426 /*@
1427   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tesselating the (x,y) plane and extruding in the third direction using wedge cells.
1428 
1429   Collective
1430 
1431   Input Parameters:
1432 + comm        - The communicator for the `DM` object
1433 . faces       - Number of faces per dimension, or NULL for (1, 1, 1)
1434 . lower       - The lower left corner, or NULL for (0, 0, 0)
1435 . upper       - The upper right corner, or NULL for (1, 1, 1)
1436 . periodicity - The boundary type for the X,Y,Z direction, or NULL for `DM_BOUNDARY_NONE`
1437 . orderHeight - If `PETSC_TRUE`, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1438 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1439 
1440   Output Parameter:
1441 . dm  - The `DM` object
1442 
1443   Level: beginner
1444 
1445 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateWedgeCylinderMesh()`, `DMExtrude()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1446 @*/
1447 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool orderHeight, PetscBool interpolate, DM *dm)
1448 {
1449   PetscInt       fac[3] = {1, 1, 1};
1450   PetscReal      low[3] = {0, 0, 0};
1451   PetscReal      upp[3] = {1, 1, 1};
1452   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1453 
1454   PetscFunctionBegin;
1455   PetscCall(DMCreate(comm, dm));
1456   PetscCall(DMSetType(*dm, DMPLEX));
1457   PetscCall(DMPlexCreateWedgeBoxMesh_Internal(*dm, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt));
1458   if (!interpolate) {
1459     DM udm;
1460 
1461     PetscCall(DMPlexUninterpolate(*dm, &udm));
1462     PetscCall(DMPlexReplace_Internal(*dm, &udm));
1463   }
1464   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1465   PetscFunctionReturn(PETSC_SUCCESS);
1466 }
1467 
1468 /*
1469   DMPlexTensorPointLexicographic_Private - Returns all tuples of size 'len' with nonnegative integers that are all less than or equal to 'max' for that dimension.
1470                                            Ordering is lexicographic with lowest index as least significant in ordering.
1471                                            e.g. for len == 2 and max == 2, this will return, in order, {0,0}, {1,0}, {2,0}, {0,1}, {1,1}, {2,1}, {0,2}, {1,2}, {2,2}.
1472 
1473   Input Parameters:
1474 + len - The length of the tuple
1475 . max - The maximum for each dimension, so values are in [0, max)
1476 - tup - A tuple of length len+1: tup[len] > 0 indicates a stopping condition
1477 
1478   Output Parameter:
1479 . tup - A tuple of len integers whose entries are at most 'max'
1480 
1481   Level: developer
1482 
1483 .seealso: PetscDualSpaceTensorPointLexicographic_Internal(), PetscDualSpaceLatticePointLexicographic_Internal()
1484 */
1485 static PetscErrorCode DMPlexTensorPointLexicographic_Private(PetscInt len, const PetscInt max[], PetscInt tup[])
1486 {
1487   PetscInt i;
1488 
1489   PetscFunctionBegin;
1490   for (i = 0; i < len; ++i) {
1491     if (tup[i] < max[i] - 1) {
1492       break;
1493     } else {
1494       tup[i] = 0;
1495     }
1496   }
1497   if (i == len) tup[i - 1] = max[i - 1];
1498   else ++tup[i];
1499   PetscFunctionReturn(PETSC_SUCCESS);
1500 }
1501 
1502 static PetscInt TupleToIndex_Private(PetscInt len, const PetscInt max[], const PetscInt tup[])
1503 {
1504   PetscInt i, idx = tup[len - 1];
1505 
1506   for (i = len - 2; i >= 0; --i) {
1507     idx *= max[i];
1508     idx += tup[i];
1509   }
1510   return idx;
1511 }
1512 
1513 static PetscErrorCode DestroyExtent_Private(void *extent)
1514 {
1515   return PetscFree(extent);
1516 }
1517 
1518 static PetscErrorCode DMPlexCreateHypercubicMesh_Internal(DM dm, PetscInt dim, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], const DMBoundaryType bd[])
1519 {
1520   Vec          coordinates;
1521   PetscSection coordSection;
1522   DMLabel      cutLabel    = NULL;
1523   PetscBool    cutMarker   = PETSC_FALSE;
1524   PetscBool    periodic    = PETSC_FALSE;
1525   PetscInt     numCells    = 1, c;
1526   PetscInt     numVertices = 1, v;
1527   PetscScalar *coords;
1528   PetscInt    *vertices, *vert, *vtmp, *supp, cone[2];
1529   PetscInt     d, e, cell = 0, coordSize;
1530   PetscMPIInt  rank;
1531 
1532   PetscFunctionBegin;
1533   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1534   PetscCall(DMSetDimension(dm, dim));
1535   PetscCall(PetscCalloc4(dim, &vertices, dim, &vert, dim, &vtmp, 2 * dim, &supp));
1536   PetscCall(DMCreateLabel(dm, "marker"));
1537   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
1538   for (d = 0; d < dim; ++d) periodic = (periodic || bd[d] == DM_BOUNDARY_PERIODIC) ? PETSC_TRUE : PETSC_FALSE;
1539   if (periodic && cutMarker) {
1540     PetscCall(DMCreateLabel(dm, "periodic_cut"));
1541     PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
1542   }
1543   for (d = 0; d < dim; ++d) PetscCheck(bd[d] == DM_BOUNDARY_PERIODIC, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Hypercubic mesh must be periodic now");
1544   for (d = 0; d < dim; ++d) {
1545     vertices[d] = edges[d];
1546     numVertices *= vertices[d];
1547   }
1548   numCells = numVertices * dim;
1549   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1550   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, 2));
1551   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetSupportSize(dm, v, 2 * dim));
1552   /* TODO Loop over boundary and reset support sizes */
1553   PetscCall(DMSetUp(dm)); /* Allocate space for cones and supports */
1554   /* Build cell cones and vertex supports */
1555   PetscCall(DMCreateLabel(dm, "celltype"));
1556   while (vert[dim - 1] < vertices[dim - 1]) {
1557     const PetscInt vertex = TupleToIndex_Private(dim, vertices, vert) + numCells;
1558     PetscInt       s      = 0;
1559 
1560     PetscCall(PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT ":", vertex));
1561     for (d = 0; d < dim; ++d) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, vert[d]));
1562     PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
1563     PetscCall(DMPlexSetCellType(dm, vertex, DM_POLYTOPE_POINT));
1564     for (d = 0; d < dim; ++d) {
1565       for (e = 0; e < dim; ++e) vtmp[e] = vert[e];
1566       vtmp[d] = (vert[d] + 1) % vertices[d];
1567       cone[0] = vertex;
1568       cone[1] = TupleToIndex_Private(dim, vertices, vtmp) + numCells;
1569       PetscCall(PetscPrintf(PETSC_COMM_SELF, "  Vertex %" PetscInt_FMT ":", cone[1]));
1570       for (e = 0; e < dim; ++e) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, vtmp[e]));
1571       PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
1572       PetscCall(DMPlexSetCone(dm, cell, cone));
1573       PetscCall(DMPlexSetCellType(dm, cell, DM_POLYTOPE_SEGMENT));
1574       PetscCall(PetscPrintf(PETSC_COMM_SELF, "  Edge %" PetscInt_FMT " (%" PetscInt_FMT " %" PetscInt_FMT ")\n", cell, cone[0], cone[1]));
1575       ++cell;
1576     }
1577     for (d = 0; d < dim; ++d) {
1578       for (e = 0; e < dim; ++e) vtmp[e] = vert[e];
1579       vtmp[d]   = (vert[d] + vertices[d] - 1) % vertices[d];
1580       supp[s++] = TupleToIndex_Private(dim, vertices, vtmp) * dim + d;
1581       supp[s++] = (vertex - numCells) * dim + d;
1582       PetscCall(DMPlexSetSupport(dm, vertex, supp));
1583     }
1584     PetscCall(DMPlexTensorPointLexicographic_Private(dim, vertices, vert));
1585   }
1586   PetscCall(DMPlexStratify(dm));
1587   /* Build coordinates */
1588   PetscCall(DMGetCoordinateSection(dm, &coordSection));
1589   PetscCall(PetscSectionSetNumFields(coordSection, 1));
1590   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1591   PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1592   for (v = numCells; v < numCells + numVertices; ++v) {
1593     PetscCall(PetscSectionSetDof(coordSection, v, dim));
1594     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1595   }
1596   PetscCall(PetscSectionSetUp(coordSection));
1597   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1598   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1599   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1600   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1601   PetscCall(VecSetBlockSize(coordinates, dim));
1602   PetscCall(VecSetType(coordinates, VECSTANDARD));
1603   PetscCall(VecGetArray(coordinates, &coords));
1604   for (d = 0; d < dim; ++d) vert[d] = 0;
1605   while (vert[dim - 1] < vertices[dim - 1]) {
1606     const PetscInt vertex = TupleToIndex_Private(dim, vertices, vert);
1607 
1608     for (d = 0; d < dim; ++d) coords[vertex * dim + d] = lower[d] + ((upper[d] - lower[d]) / vertices[d]) * vert[d];
1609     PetscCall(PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT ":", vertex));
1610     for (d = 0; d < dim; ++d) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, vert[d]));
1611     for (d = 0; d < dim; ++d) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %g", (double)PetscRealPart(coords[vertex * dim + d])));
1612     PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
1613     PetscCall(DMPlexTensorPointLexicographic_Private(dim, vertices, vert));
1614   }
1615   PetscCall(VecRestoreArray(coordinates, &coords));
1616   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1617   PetscCall(VecDestroy(&coordinates));
1618   PetscCall(PetscFree4(vertices, vert, vtmp, supp));
1619   //PetscCall(DMSetPeriodicity(dm, NULL, lower, upper));
1620   // Attach the extent
1621   {
1622     PetscContainer c;
1623     PetscInt      *extent;
1624 
1625     PetscCall(PetscMalloc1(dim, &extent));
1626     for (PetscInt d = 0; d < dim; ++d) extent[d] = edges[d];
1627     PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &c));
1628     PetscCall(PetscContainerSetUserDestroy(c, DestroyExtent_Private));
1629     PetscCall(PetscContainerSetPointer(c, extent));
1630     PetscCall(PetscObjectCompose((PetscObject)dm, "_extent", (PetscObject)c));
1631     PetscCall(PetscContainerDestroy(&c));
1632   }
1633   PetscFunctionReturn(PETSC_SUCCESS);
1634 }
1635 
1636 /*@C
1637   DMPlexCreateHypercubicMesh - Creates a peridoic mesh on the tensor product of unit intervals using only vertices and edges.
1638 
1639   Collective
1640 
1641   Input Parameters:
1642 + comm        - The communicator for the DM object
1643 . dim         - The spatial dimension
1644 . edges       - Number of edges per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1645 . lower       - The lower left corner, or NULL for (0, 0, 0)
1646 - upper       - The upper right corner, or NULL for (1, 1, 1)
1647 
1648   Output Parameter:
1649 . dm  - The DM object
1650 
1651   Note: If you want to customize this mesh using options, you just need to
1652 $  DMCreate(comm, &dm);
1653 $  DMSetType(dm, DMPLEX);
1654 $  DMSetFromOptions(dm);
1655 and use the options on the DMSetFromOptions() page.
1656 
1657   The vertices are numbered is lexicographic order, and the dim edges exiting a vertex in the positive orthant are number consecutively,
1658 $ 18--0-19--2-20--4-18
1659 $  |     |     |     |
1660 $ 13    15    17    13
1661 $  |     |     |     |
1662 $ 24-12-25-14-26-16-24
1663 $  |     |     |     |
1664 $  7     9    11     7
1665 $  |     |     |     |
1666 $ 21--6-22--8-23-10-21
1667 $  |     |     |     |
1668 $  1     3     5     1
1669 $  |     |     |     |
1670 $ 18--0-19--2-20--4-18
1671 
1672   Level: beginner
1673 
1674 .seealso: DMSetFromOptions(), DMPlexCreateFromFile(), DMPlexCreateHexCylinderMesh(), DMSetType(), DMCreate()
1675 @*/
1676 PetscErrorCode DMPlexCreateHypercubicMesh(MPI_Comm comm, PetscInt dim, const PetscInt edges[], const PetscReal lower[], const PetscReal upper[], DM *dm)
1677 {
1678   PetscInt       *edg;
1679   PetscReal      *low, *upp;
1680   DMBoundaryType *bdt;
1681   PetscInt        d;
1682 
1683   PetscFunctionBegin;
1684   PetscCall(DMCreate(comm, dm));
1685   PetscCall(DMSetType(*dm, DMPLEX));
1686   PetscCall(PetscMalloc4(dim, &edg, dim, &low, dim, &upp, dim, &bdt));
1687   for (d = 0; d < dim; ++d) {
1688     edg[d] = edges ? edges[d] : 1;
1689     low[d] = lower ? lower[d] : 0.;
1690     upp[d] = upper ? upper[d] : 1.;
1691     bdt[d] = DM_BOUNDARY_PERIODIC;
1692   }
1693   PetscCall(DMPlexCreateHypercubicMesh_Internal(*dm, dim, low, upp, edg, bdt));
1694   PetscCall(PetscFree4(edg, low, upp, bdt));
1695   PetscFunctionReturn(PETSC_SUCCESS);
1696 }
1697 
1698 /*@C
1699   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all `DM` options in the database.
1700 
1701   Logically Collective on dm
1702 
1703   Input Parameters:
1704 + dm - the DM context
1705 - prefix - the prefix to prepend to all option names
1706 
1707   Level: advanced
1708 
1709   Note:
1710   A hyphen (-) must NOT be given at the beginning of the prefix name.
1711   The first character of all runtime options is AUTOMATICALLY the hyphen.
1712 
1713 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `SNESSetFromOptions()`
1714 @*/
1715 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
1716 {
1717   DM_Plex *mesh = (DM_Plex *)dm->data;
1718 
1719   PetscFunctionBegin;
1720   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1721   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, prefix));
1722   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)mesh->partitioner, prefix));
1723   PetscFunctionReturn(PETSC_SUCCESS);
1724 }
1725 
1726 /* Remap geometry to cylinder
1727    TODO: This only works for a single refinement, then it is broken
1728 
1729      Interior square: Linear interpolation is correct
1730      The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
1731      such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
1732 
1733        phi     = arctan(y/x)
1734        d_close = sqrt(1/8 + 1/4 sin^2(phi))
1735        d_far   = sqrt(1/2 + sin^2(phi))
1736 
1737      so we remap them using
1738 
1739        x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
1740        y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
1741 
1742      If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
1743 */
1744 static void snapToCylinder(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
1745 {
1746   const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1747   const PetscReal ds2 = 0.5 * dis;
1748 
1749   if ((PetscAbsScalar(u[0]) <= ds2) && (PetscAbsScalar(u[1]) <= ds2)) {
1750     f0[0] = u[0];
1751     f0[1] = u[1];
1752   } else {
1753     PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
1754 
1755     x    = PetscRealPart(u[0]);
1756     y    = PetscRealPart(u[1]);
1757     phi  = PetscAtan2Real(y, x);
1758     sinp = PetscSinReal(phi);
1759     cosp = PetscCosReal(phi);
1760     if ((PetscAbsReal(phi) > PETSC_PI / 4.0) && (PetscAbsReal(phi) < 3.0 * PETSC_PI / 4.0)) {
1761       dc = PetscAbsReal(ds2 / sinp);
1762       df = PetscAbsReal(dis / sinp);
1763       xc = ds2 * x / PetscAbsReal(y);
1764       yc = ds2 * PetscSignReal(y);
1765     } else {
1766       dc = PetscAbsReal(ds2 / cosp);
1767       df = PetscAbsReal(dis / cosp);
1768       xc = ds2 * PetscSignReal(x);
1769       yc = ds2 * y / PetscAbsReal(x);
1770     }
1771     f0[0] = xc + (u[0] - xc) * (1.0 - dc) / (df - dc);
1772     f0[1] = yc + (u[1] - yc) * (1.0 - dc) / (df - dc);
1773   }
1774   f0[2] = u[2];
1775 }
1776 
1777 static PetscErrorCode DMPlexCreateHexCylinderMesh_Internal(DM dm, DMBoundaryType periodicZ)
1778 {
1779   const PetscInt dim = 3;
1780   PetscInt       numCells, numVertices;
1781   PetscMPIInt    rank;
1782 
1783   PetscFunctionBegin;
1784   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
1785   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1786   PetscCall(DMSetDimension(dm, dim));
1787   /* Create topology */
1788   {
1789     PetscInt cone[8], c;
1790 
1791     numCells    = rank == 0 ? 5 : 0;
1792     numVertices = rank == 0 ? 16 : 0;
1793     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1794       numCells *= 3;
1795       numVertices = rank == 0 ? 24 : 0;
1796     }
1797     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1798     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 8));
1799     PetscCall(DMSetUp(dm));
1800     if (rank == 0) {
1801       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1802         cone[0] = 15;
1803         cone[1] = 18;
1804         cone[2] = 17;
1805         cone[3] = 16;
1806         cone[4] = 31;
1807         cone[5] = 32;
1808         cone[6] = 33;
1809         cone[7] = 34;
1810         PetscCall(DMPlexSetCone(dm, 0, cone));
1811         cone[0] = 16;
1812         cone[1] = 17;
1813         cone[2] = 24;
1814         cone[3] = 23;
1815         cone[4] = 32;
1816         cone[5] = 36;
1817         cone[6] = 37;
1818         cone[7] = 33; /* 22 25 26 21 */
1819         PetscCall(DMPlexSetCone(dm, 1, cone));
1820         cone[0] = 18;
1821         cone[1] = 27;
1822         cone[2] = 24;
1823         cone[3] = 17;
1824         cone[4] = 34;
1825         cone[5] = 33;
1826         cone[6] = 37;
1827         cone[7] = 38;
1828         PetscCall(DMPlexSetCone(dm, 2, cone));
1829         cone[0] = 29;
1830         cone[1] = 27;
1831         cone[2] = 18;
1832         cone[3] = 15;
1833         cone[4] = 35;
1834         cone[5] = 31;
1835         cone[6] = 34;
1836         cone[7] = 38;
1837         PetscCall(DMPlexSetCone(dm, 3, cone));
1838         cone[0] = 29;
1839         cone[1] = 15;
1840         cone[2] = 16;
1841         cone[3] = 23;
1842         cone[4] = 35;
1843         cone[5] = 36;
1844         cone[6] = 32;
1845         cone[7] = 31;
1846         PetscCall(DMPlexSetCone(dm, 4, cone));
1847 
1848         cone[0] = 31;
1849         cone[1] = 34;
1850         cone[2] = 33;
1851         cone[3] = 32;
1852         cone[4] = 19;
1853         cone[5] = 22;
1854         cone[6] = 21;
1855         cone[7] = 20;
1856         PetscCall(DMPlexSetCone(dm, 5, cone));
1857         cone[0] = 32;
1858         cone[1] = 33;
1859         cone[2] = 37;
1860         cone[3] = 36;
1861         cone[4] = 22;
1862         cone[5] = 25;
1863         cone[6] = 26;
1864         cone[7] = 21;
1865         PetscCall(DMPlexSetCone(dm, 6, cone));
1866         cone[0] = 34;
1867         cone[1] = 38;
1868         cone[2] = 37;
1869         cone[3] = 33;
1870         cone[4] = 20;
1871         cone[5] = 21;
1872         cone[6] = 26;
1873         cone[7] = 28;
1874         PetscCall(DMPlexSetCone(dm, 7, cone));
1875         cone[0] = 35;
1876         cone[1] = 38;
1877         cone[2] = 34;
1878         cone[3] = 31;
1879         cone[4] = 30;
1880         cone[5] = 19;
1881         cone[6] = 20;
1882         cone[7] = 28;
1883         PetscCall(DMPlexSetCone(dm, 8, cone));
1884         cone[0] = 35;
1885         cone[1] = 31;
1886         cone[2] = 32;
1887         cone[3] = 36;
1888         cone[4] = 30;
1889         cone[5] = 25;
1890         cone[6] = 22;
1891         cone[7] = 19;
1892         PetscCall(DMPlexSetCone(dm, 9, cone));
1893 
1894         cone[0] = 19;
1895         cone[1] = 20;
1896         cone[2] = 21;
1897         cone[3] = 22;
1898         cone[4] = 15;
1899         cone[5] = 16;
1900         cone[6] = 17;
1901         cone[7] = 18;
1902         PetscCall(DMPlexSetCone(dm, 10, cone));
1903         cone[0] = 22;
1904         cone[1] = 21;
1905         cone[2] = 26;
1906         cone[3] = 25;
1907         cone[4] = 16;
1908         cone[5] = 23;
1909         cone[6] = 24;
1910         cone[7] = 17;
1911         PetscCall(DMPlexSetCone(dm, 11, cone));
1912         cone[0] = 20;
1913         cone[1] = 28;
1914         cone[2] = 26;
1915         cone[3] = 21;
1916         cone[4] = 18;
1917         cone[5] = 17;
1918         cone[6] = 24;
1919         cone[7] = 27;
1920         PetscCall(DMPlexSetCone(dm, 12, cone));
1921         cone[0] = 30;
1922         cone[1] = 28;
1923         cone[2] = 20;
1924         cone[3] = 19;
1925         cone[4] = 29;
1926         cone[5] = 15;
1927         cone[6] = 18;
1928         cone[7] = 27;
1929         PetscCall(DMPlexSetCone(dm, 13, cone));
1930         cone[0] = 30;
1931         cone[1] = 19;
1932         cone[2] = 22;
1933         cone[3] = 25;
1934         cone[4] = 29;
1935         cone[5] = 23;
1936         cone[6] = 16;
1937         cone[7] = 15;
1938         PetscCall(DMPlexSetCone(dm, 14, cone));
1939       } else {
1940         cone[0] = 5;
1941         cone[1] = 8;
1942         cone[2] = 7;
1943         cone[3] = 6;
1944         cone[4] = 9;
1945         cone[5] = 12;
1946         cone[6] = 11;
1947         cone[7] = 10;
1948         PetscCall(DMPlexSetCone(dm, 0, cone));
1949         cone[0] = 6;
1950         cone[1] = 7;
1951         cone[2] = 14;
1952         cone[3] = 13;
1953         cone[4] = 12;
1954         cone[5] = 15;
1955         cone[6] = 16;
1956         cone[7] = 11;
1957         PetscCall(DMPlexSetCone(dm, 1, cone));
1958         cone[0] = 8;
1959         cone[1] = 17;
1960         cone[2] = 14;
1961         cone[3] = 7;
1962         cone[4] = 10;
1963         cone[5] = 11;
1964         cone[6] = 16;
1965         cone[7] = 18;
1966         PetscCall(DMPlexSetCone(dm, 2, cone));
1967         cone[0] = 19;
1968         cone[1] = 17;
1969         cone[2] = 8;
1970         cone[3] = 5;
1971         cone[4] = 20;
1972         cone[5] = 9;
1973         cone[6] = 10;
1974         cone[7] = 18;
1975         PetscCall(DMPlexSetCone(dm, 3, cone));
1976         cone[0] = 19;
1977         cone[1] = 5;
1978         cone[2] = 6;
1979         cone[3] = 13;
1980         cone[4] = 20;
1981         cone[5] = 15;
1982         cone[6] = 12;
1983         cone[7] = 9;
1984         PetscCall(DMPlexSetCone(dm, 4, cone));
1985       }
1986     }
1987     PetscCall(DMPlexSymmetrize(dm));
1988     PetscCall(DMPlexStratify(dm));
1989   }
1990   /* Create cube geometry */
1991   {
1992     Vec             coordinates;
1993     PetscSection    coordSection;
1994     PetscScalar    *coords;
1995     PetscInt        coordSize, v;
1996     const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1997     const PetscReal ds2 = dis / 2.0;
1998 
1999     /* Build coordinates */
2000     PetscCall(DMGetCoordinateSection(dm, &coordSection));
2001     PetscCall(PetscSectionSetNumFields(coordSection, 1));
2002     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
2003     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
2004     for (v = numCells; v < numCells + numVertices; ++v) {
2005       PetscCall(PetscSectionSetDof(coordSection, v, dim));
2006       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
2007     }
2008     PetscCall(PetscSectionSetUp(coordSection));
2009     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2010     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2011     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2012     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2013     PetscCall(VecSetBlockSize(coordinates, dim));
2014     PetscCall(VecSetType(coordinates, VECSTANDARD));
2015     PetscCall(VecGetArray(coordinates, &coords));
2016     if (rank == 0) {
2017       coords[0 * dim + 0]  = -ds2;
2018       coords[0 * dim + 1]  = -ds2;
2019       coords[0 * dim + 2]  = 0.0;
2020       coords[1 * dim + 0]  = ds2;
2021       coords[1 * dim + 1]  = -ds2;
2022       coords[1 * dim + 2]  = 0.0;
2023       coords[2 * dim + 0]  = ds2;
2024       coords[2 * dim + 1]  = ds2;
2025       coords[2 * dim + 2]  = 0.0;
2026       coords[3 * dim + 0]  = -ds2;
2027       coords[3 * dim + 1]  = ds2;
2028       coords[3 * dim + 2]  = 0.0;
2029       coords[4 * dim + 0]  = -ds2;
2030       coords[4 * dim + 1]  = -ds2;
2031       coords[4 * dim + 2]  = 1.0;
2032       coords[5 * dim + 0]  = -ds2;
2033       coords[5 * dim + 1]  = ds2;
2034       coords[5 * dim + 2]  = 1.0;
2035       coords[6 * dim + 0]  = ds2;
2036       coords[6 * dim + 1]  = ds2;
2037       coords[6 * dim + 2]  = 1.0;
2038       coords[7 * dim + 0]  = ds2;
2039       coords[7 * dim + 1]  = -ds2;
2040       coords[7 * dim + 2]  = 1.0;
2041       coords[8 * dim + 0]  = dis;
2042       coords[8 * dim + 1]  = -dis;
2043       coords[8 * dim + 2]  = 0.0;
2044       coords[9 * dim + 0]  = dis;
2045       coords[9 * dim + 1]  = dis;
2046       coords[9 * dim + 2]  = 0.0;
2047       coords[10 * dim + 0] = dis;
2048       coords[10 * dim + 1] = -dis;
2049       coords[10 * dim + 2] = 1.0;
2050       coords[11 * dim + 0] = dis;
2051       coords[11 * dim + 1] = dis;
2052       coords[11 * dim + 2] = 1.0;
2053       coords[12 * dim + 0] = -dis;
2054       coords[12 * dim + 1] = dis;
2055       coords[12 * dim + 2] = 0.0;
2056       coords[13 * dim + 0] = -dis;
2057       coords[13 * dim + 1] = dis;
2058       coords[13 * dim + 2] = 1.0;
2059       coords[14 * dim + 0] = -dis;
2060       coords[14 * dim + 1] = -dis;
2061       coords[14 * dim + 2] = 0.0;
2062       coords[15 * dim + 0] = -dis;
2063       coords[15 * dim + 1] = -dis;
2064       coords[15 * dim + 2] = 1.0;
2065       if (periodicZ == DM_BOUNDARY_PERIODIC) {
2066         /* 15 31 19 */ coords[16 * dim + 0] = -ds2;
2067         coords[16 * dim + 1]                = -ds2;
2068         coords[16 * dim + 2]                = 0.5;
2069         /* 16 32 22 */ coords[17 * dim + 0] = ds2;
2070         coords[17 * dim + 1]                = -ds2;
2071         coords[17 * dim + 2]                = 0.5;
2072         /* 17 33 21 */ coords[18 * dim + 0] = ds2;
2073         coords[18 * dim + 1]                = ds2;
2074         coords[18 * dim + 2]                = 0.5;
2075         /* 18 34 20 */ coords[19 * dim + 0] = -ds2;
2076         coords[19 * dim + 1]                = ds2;
2077         coords[19 * dim + 2]                = 0.5;
2078         /* 29 35 30 */ coords[20 * dim + 0] = -dis;
2079         coords[20 * dim + 1]                = -dis;
2080         coords[20 * dim + 2]                = 0.5;
2081         /* 23 36 25 */ coords[21 * dim + 0] = dis;
2082         coords[21 * dim + 1]                = -dis;
2083         coords[21 * dim + 2]                = 0.5;
2084         /* 24 37 26 */ coords[22 * dim + 0] = dis;
2085         coords[22 * dim + 1]                = dis;
2086         coords[22 * dim + 2]                = 0.5;
2087         /* 27 38 28 */ coords[23 * dim + 0] = -dis;
2088         coords[23 * dim + 1]                = dis;
2089         coords[23 * dim + 2]                = 0.5;
2090       }
2091     }
2092     PetscCall(VecRestoreArray(coordinates, &coords));
2093     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2094     PetscCall(VecDestroy(&coordinates));
2095   }
2096   /* Create periodicity */
2097   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
2098     PetscReal L[3]       = {-1., -1., 0.};
2099     PetscReal maxCell[3] = {-1., -1., 0.};
2100     PetscReal lower[3]   = {0.0, 0.0, 0.0};
2101     PetscReal upper[3]   = {1.0, 1.0, 1.5};
2102     PetscInt  numZCells  = 3;
2103 
2104     L[2]       = upper[2] - lower[2];
2105     maxCell[2] = 1.1 * (L[2] / numZCells);
2106     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
2107   }
2108   {
2109     DM          cdm;
2110     PetscDS     cds;
2111     PetscScalar c[2] = {1.0, 1.0};
2112 
2113     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToCylinder));
2114     PetscCall(DMGetCoordinateDM(dm, &cdm));
2115     PetscCall(DMGetDS(cdm, &cds));
2116     PetscCall(PetscDSSetConstants(cds, 2, c));
2117   }
2118   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
2119 
2120   /* Wait for coordinate creation before doing in-place modification */
2121   PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2122   PetscFunctionReturn(PETSC_SUCCESS);
2123 }
2124 
2125 /*@
2126   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
2127 
2128   Collective
2129 
2130   Input Parameters:
2131 + comm      - The communicator for the `DM` object
2132 - periodicZ - The boundary type for the Z direction
2133 
2134   Output Parameter:
2135 . dm  - The DM object
2136 
2137   Level: beginner
2138 
2139   Note:
2140   Here is the output numbering looking from the bottom of the cylinder:
2141 .vb
2142        17-----14
2143         |     |
2144         |  2  |
2145         |     |
2146  17-----8-----7-----14
2147   |     |     |     |
2148   |  3  |  0  |  1  |
2149   |     |     |     |
2150  19-----5-----6-----13
2151         |     |
2152         |  4  |
2153         |     |
2154        19-----13
2155 
2156  and up through the top
2157 
2158        18-----16
2159         |     |
2160         |  2  |
2161         |     |
2162  18----10----11-----16
2163   |     |     |     |
2164   |  3  |  0  |  1  |
2165   |     |     |     |
2166  20-----9----12-----15
2167         |     |
2168         |  4  |
2169         |     |
2170        20-----15
2171 .ve
2172 
2173 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2174 @*/
2175 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, DMBoundaryType periodicZ, DM *dm)
2176 {
2177   PetscFunctionBegin;
2178   PetscValidPointer(dm, 3);
2179   PetscCall(DMCreate(comm, dm));
2180   PetscCall(DMSetType(*dm, DMPLEX));
2181   PetscCall(DMPlexCreateHexCylinderMesh_Internal(*dm, periodicZ));
2182   PetscFunctionReturn(PETSC_SUCCESS);
2183 }
2184 
2185 static PetscErrorCode DMPlexCreateWedgeCylinderMesh_Internal(DM dm, PetscInt n, PetscBool interpolate)
2186 {
2187   const PetscInt dim = 3;
2188   PetscInt       numCells, numVertices, v;
2189   PetscMPIInt    rank;
2190 
2191   PetscFunctionBegin;
2192   PetscCheck(n >= 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %" PetscInt_FMT " cannot be negative", n);
2193   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
2194   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2195   PetscCall(DMSetDimension(dm, dim));
2196   /* Must create the celltype label here so that we do not automatically try to compute the types */
2197   PetscCall(DMCreateLabel(dm, "celltype"));
2198   /* Create topology */
2199   {
2200     PetscInt cone[6], c;
2201 
2202     numCells    = rank == 0 ? n : 0;
2203     numVertices = rank == 0 ? 2 * (n + 1) : 0;
2204     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
2205     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
2206     PetscCall(DMSetUp(dm));
2207     for (c = 0; c < numCells; c++) {
2208       cone[0] = c + n * 1;
2209       cone[1] = (c + 1) % n + n * 1;
2210       cone[2] = 0 + 3 * n;
2211       cone[3] = c + n * 2;
2212       cone[4] = (c + 1) % n + n * 2;
2213       cone[5] = 1 + 3 * n;
2214       PetscCall(DMPlexSetCone(dm, c, cone));
2215       PetscCall(DMPlexSetCellType(dm, c, DM_POLYTOPE_TRI_PRISM_TENSOR));
2216     }
2217     PetscCall(DMPlexSymmetrize(dm));
2218     PetscCall(DMPlexStratify(dm));
2219   }
2220   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetCellType(dm, v, DM_POLYTOPE_POINT));
2221   /* Create cylinder geometry */
2222   {
2223     Vec          coordinates;
2224     PetscSection coordSection;
2225     PetscScalar *coords;
2226     PetscInt     coordSize, c;
2227 
2228     /* Build coordinates */
2229     PetscCall(DMGetCoordinateSection(dm, &coordSection));
2230     PetscCall(PetscSectionSetNumFields(coordSection, 1));
2231     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
2232     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
2233     for (v = numCells; v < numCells + numVertices; ++v) {
2234       PetscCall(PetscSectionSetDof(coordSection, v, dim));
2235       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
2236     }
2237     PetscCall(PetscSectionSetUp(coordSection));
2238     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2239     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2240     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2241     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2242     PetscCall(VecSetBlockSize(coordinates, dim));
2243     PetscCall(VecSetType(coordinates, VECSTANDARD));
2244     PetscCall(VecGetArray(coordinates, &coords));
2245     for (c = 0; c < numCells; c++) {
2246       coords[(c + 0 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
2247       coords[(c + 0 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
2248       coords[(c + 0 * n) * dim + 2] = 1.0;
2249       coords[(c + 1 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
2250       coords[(c + 1 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
2251       coords[(c + 1 * n) * dim + 2] = 0.0;
2252     }
2253     if (rank == 0) {
2254       coords[(2 * n + 0) * dim + 0] = 0.0;
2255       coords[(2 * n + 0) * dim + 1] = 0.0;
2256       coords[(2 * n + 0) * dim + 2] = 1.0;
2257       coords[(2 * n + 1) * dim + 0] = 0.0;
2258       coords[(2 * n + 1) * dim + 1] = 0.0;
2259       coords[(2 * n + 1) * dim + 2] = 0.0;
2260     }
2261     PetscCall(VecRestoreArray(coordinates, &coords));
2262     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2263     PetscCall(VecDestroy(&coordinates));
2264   }
2265   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
2266   /* Interpolate */
2267   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2268   PetscFunctionReturn(PETSC_SUCCESS);
2269 }
2270 
2271 /*@
2272   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
2273 
2274   Collective
2275 
2276   Input Parameters:
2277 + comm - The communicator for the `DM` object
2278 . n    - The number of wedges around the origin
2279 - interpolate - Create edges and faces
2280 
2281   Output Parameter:
2282 . dm  - The `DM` object
2283 
2284   Level: beginner
2285 
2286 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2287 @*/
2288 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
2289 {
2290   PetscFunctionBegin;
2291   PetscValidPointer(dm, 4);
2292   PetscCall(DMCreate(comm, dm));
2293   PetscCall(DMSetType(*dm, DMPLEX));
2294   PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(*dm, n, interpolate));
2295   PetscFunctionReturn(PETSC_SUCCESS);
2296 }
2297 
2298 static inline PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2299 {
2300   PetscReal prod = 0.0;
2301   PetscInt  i;
2302   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
2303   return PetscSqrtReal(prod);
2304 }
2305 static inline PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2306 {
2307   PetscReal prod = 0.0;
2308   PetscInt  i;
2309   for (i = 0; i < dim; ++i) prod += x[i] * y[i];
2310   return prod;
2311 }
2312 
2313 /* The first constant is the sphere radius */
2314 static void snapToSphere(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
2315 {
2316   PetscReal r     = PetscRealPart(constants[0]);
2317   PetscReal norm2 = 0.0, fac;
2318   PetscInt  n     = uOff[1] - uOff[0], d;
2319 
2320   for (d = 0; d < n; ++d) norm2 += PetscSqr(PetscRealPart(u[d]));
2321   fac = r / PetscSqrtReal(norm2);
2322   for (d = 0; d < n; ++d) f0[d] = u[d] * fac;
2323 }
2324 
2325 static PetscErrorCode DMPlexCreateSphereMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, PetscReal R)
2326 {
2327   const PetscInt embedDim = dim + 1;
2328   PetscSection   coordSection;
2329   Vec            coordinates;
2330   PetscScalar   *coords;
2331   PetscReal     *coordsIn;
2332   PetscInt       numCells, numEdges, numVerts = 0, firstVertex = 0, v, firstEdge, coordSize, d, c, e;
2333   PetscMPIInt    rank;
2334 
2335   PetscFunctionBegin;
2336   PetscValidLogicalCollectiveBool(dm, simplex, 3);
2337   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
2338   PetscCall(DMSetDimension(dm, dim));
2339   PetscCall(DMSetCoordinateDim(dm, dim + 1));
2340   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2341   switch (dim) {
2342   case 2:
2343     if (simplex) {
2344       const PetscReal radius    = PetscSqrtReal(1 + PETSC_PHI * PETSC_PHI) / (1.0 + PETSC_PHI);
2345       const PetscReal edgeLen   = 2.0 / (1.0 + PETSC_PHI) * (R / radius);
2346       const PetscInt  degree    = 5;
2347       PetscReal       vertex[3] = {0.0, 1.0 / (1.0 + PETSC_PHI), PETSC_PHI / (1.0 + PETSC_PHI)};
2348       PetscInt        s[3]      = {1, 1, 1};
2349       PetscInt        cone[3];
2350       PetscInt       *graph, p, i, j, k;
2351 
2352       vertex[0] *= R / radius;
2353       vertex[1] *= R / radius;
2354       vertex[2] *= R / radius;
2355       numCells    = rank == 0 ? 20 : 0;
2356       numVerts    = rank == 0 ? 12 : 0;
2357       firstVertex = numCells;
2358       /* Use icosahedron, which for a R-sphere has coordinates which are all cyclic permutations of
2359 
2360            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
2361 
2362          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2363          length is then given by 2/(1+\phi) = 2 * 0.38197 = 0.76393.
2364       */
2365       /* Construct vertices */
2366       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2367       if (rank == 0) {
2368         for (p = 0, i = 0; p < embedDim; ++p) {
2369           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2370             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2371               for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertex[(d + p) % embedDim];
2372               ++i;
2373             }
2374           }
2375         }
2376       }
2377       /* Construct graph */
2378       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2379       for (i = 0; i < numVerts; ++i) {
2380         for (j = 0, k = 0; j < numVerts; ++j) {
2381           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2382             graph[i * numVerts + j] = 1;
2383             ++k;
2384           }
2385         }
2386         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid icosahedron, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2387       }
2388       /* Build Topology */
2389       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2390       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2391       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2392       /* Cells */
2393       for (i = 0, c = 0; i < numVerts; ++i) {
2394         for (j = 0; j < i; ++j) {
2395           for (k = 0; k < j; ++k) {
2396             if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i]) {
2397               cone[0] = firstVertex + i;
2398               cone[1] = firstVertex + j;
2399               cone[2] = firstVertex + k;
2400               /* Check orientation */
2401               {
2402                 const PetscInt epsilon[3][3][3] = {
2403                   {{0, 0, 0},  {0, 0, 1},  {0, -1, 0}},
2404                   {{0, 0, -1}, {0, 0, 0},  {1, 0, 0} },
2405                   {{0, 1, 0},  {-1, 0, 0}, {0, 0, 0} }
2406                 };
2407                 PetscReal normal[3];
2408                 PetscInt  e, f;
2409 
2410                 for (d = 0; d < embedDim; ++d) {
2411                   normal[d] = 0.0;
2412                   for (e = 0; e < embedDim; ++e) {
2413                     for (f = 0; f < embedDim; ++f) normal[d] += epsilon[d][e][f] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]);
2414                   }
2415                 }
2416                 if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2417                   PetscInt tmp = cone[1];
2418                   cone[1]      = cone[2];
2419                   cone[2]      = tmp;
2420                 }
2421               }
2422               PetscCall(DMPlexSetCone(dm, c++, cone));
2423             }
2424           }
2425         }
2426       }
2427       PetscCall(DMPlexSymmetrize(dm));
2428       PetscCall(DMPlexStratify(dm));
2429       PetscCall(PetscFree(graph));
2430     } else {
2431       /*
2432         12-21--13
2433          |     |
2434         25  4  24
2435          |     |
2436   12-25--9-16--8-24--13
2437    |     |     |     |
2438   23  5 17  0 15  3  22
2439    |     |     |     |
2440   10-20--6-14--7-19--11
2441          |     |
2442         20  1  19
2443          |     |
2444         10-18--11
2445          |     |
2446         23  2  22
2447          |     |
2448         12-21--13
2449        */
2450       PetscInt cone[4], ornt[4];
2451 
2452       numCells    = rank == 0 ? 6 : 0;
2453       numEdges    = rank == 0 ? 12 : 0;
2454       numVerts    = rank == 0 ? 8 : 0;
2455       firstVertex = numCells;
2456       firstEdge   = numCells + numVerts;
2457       /* Build Topology */
2458       PetscCall(DMPlexSetChart(dm, 0, numCells + numEdges + numVerts));
2459       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 4));
2460       for (e = firstEdge; e < firstEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
2461       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2462       if (rank == 0) {
2463         /* Cell 0 */
2464         cone[0] = 14;
2465         cone[1] = 15;
2466         cone[2] = 16;
2467         cone[3] = 17;
2468         PetscCall(DMPlexSetCone(dm, 0, cone));
2469         ornt[0] = 0;
2470         ornt[1] = 0;
2471         ornt[2] = 0;
2472         ornt[3] = 0;
2473         PetscCall(DMPlexSetConeOrientation(dm, 0, ornt));
2474         /* Cell 1 */
2475         cone[0] = 18;
2476         cone[1] = 19;
2477         cone[2] = 14;
2478         cone[3] = 20;
2479         PetscCall(DMPlexSetCone(dm, 1, cone));
2480         ornt[0] = 0;
2481         ornt[1] = 0;
2482         ornt[2] = -1;
2483         ornt[3] = 0;
2484         PetscCall(DMPlexSetConeOrientation(dm, 1, ornt));
2485         /* Cell 2 */
2486         cone[0] = 21;
2487         cone[1] = 22;
2488         cone[2] = 18;
2489         cone[3] = 23;
2490         PetscCall(DMPlexSetCone(dm, 2, cone));
2491         ornt[0] = 0;
2492         ornt[1] = 0;
2493         ornt[2] = -1;
2494         ornt[3] = 0;
2495         PetscCall(DMPlexSetConeOrientation(dm, 2, ornt));
2496         /* Cell 3 */
2497         cone[0] = 19;
2498         cone[1] = 22;
2499         cone[2] = 24;
2500         cone[3] = 15;
2501         PetscCall(DMPlexSetCone(dm, 3, cone));
2502         ornt[0] = -1;
2503         ornt[1] = -1;
2504         ornt[2] = 0;
2505         ornt[3] = -1;
2506         PetscCall(DMPlexSetConeOrientation(dm, 3, ornt));
2507         /* Cell 4 */
2508         cone[0] = 16;
2509         cone[1] = 24;
2510         cone[2] = 21;
2511         cone[3] = 25;
2512         PetscCall(DMPlexSetCone(dm, 4, cone));
2513         ornt[0] = -1;
2514         ornt[1] = -1;
2515         ornt[2] = -1;
2516         ornt[3] = 0;
2517         PetscCall(DMPlexSetConeOrientation(dm, 4, ornt));
2518         /* Cell 5 */
2519         cone[0] = 20;
2520         cone[1] = 17;
2521         cone[2] = 25;
2522         cone[3] = 23;
2523         PetscCall(DMPlexSetCone(dm, 5, cone));
2524         ornt[0] = -1;
2525         ornt[1] = -1;
2526         ornt[2] = -1;
2527         ornt[3] = -1;
2528         PetscCall(DMPlexSetConeOrientation(dm, 5, ornt));
2529         /* Edges */
2530         cone[0] = 6;
2531         cone[1] = 7;
2532         PetscCall(DMPlexSetCone(dm, 14, cone));
2533         cone[0] = 7;
2534         cone[1] = 8;
2535         PetscCall(DMPlexSetCone(dm, 15, cone));
2536         cone[0] = 8;
2537         cone[1] = 9;
2538         PetscCall(DMPlexSetCone(dm, 16, cone));
2539         cone[0] = 9;
2540         cone[1] = 6;
2541         PetscCall(DMPlexSetCone(dm, 17, cone));
2542         cone[0] = 10;
2543         cone[1] = 11;
2544         PetscCall(DMPlexSetCone(dm, 18, cone));
2545         cone[0] = 11;
2546         cone[1] = 7;
2547         PetscCall(DMPlexSetCone(dm, 19, cone));
2548         cone[0] = 6;
2549         cone[1] = 10;
2550         PetscCall(DMPlexSetCone(dm, 20, cone));
2551         cone[0] = 12;
2552         cone[1] = 13;
2553         PetscCall(DMPlexSetCone(dm, 21, cone));
2554         cone[0] = 13;
2555         cone[1] = 11;
2556         PetscCall(DMPlexSetCone(dm, 22, cone));
2557         cone[0] = 10;
2558         cone[1] = 12;
2559         PetscCall(DMPlexSetCone(dm, 23, cone));
2560         cone[0] = 13;
2561         cone[1] = 8;
2562         PetscCall(DMPlexSetCone(dm, 24, cone));
2563         cone[0] = 12;
2564         cone[1] = 9;
2565         PetscCall(DMPlexSetCone(dm, 25, cone));
2566       }
2567       PetscCall(DMPlexSymmetrize(dm));
2568       PetscCall(DMPlexStratify(dm));
2569       /* Build coordinates */
2570       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2571       if (rank == 0) {
2572         coordsIn[0 * embedDim + 0] = -R;
2573         coordsIn[0 * embedDim + 1] = R;
2574         coordsIn[0 * embedDim + 2] = -R;
2575         coordsIn[1 * embedDim + 0] = R;
2576         coordsIn[1 * embedDim + 1] = R;
2577         coordsIn[1 * embedDim + 2] = -R;
2578         coordsIn[2 * embedDim + 0] = R;
2579         coordsIn[2 * embedDim + 1] = -R;
2580         coordsIn[2 * embedDim + 2] = -R;
2581         coordsIn[3 * embedDim + 0] = -R;
2582         coordsIn[3 * embedDim + 1] = -R;
2583         coordsIn[3 * embedDim + 2] = -R;
2584         coordsIn[4 * embedDim + 0] = -R;
2585         coordsIn[4 * embedDim + 1] = R;
2586         coordsIn[4 * embedDim + 2] = R;
2587         coordsIn[5 * embedDim + 0] = R;
2588         coordsIn[5 * embedDim + 1] = R;
2589         coordsIn[5 * embedDim + 2] = R;
2590         coordsIn[6 * embedDim + 0] = -R;
2591         coordsIn[6 * embedDim + 1] = -R;
2592         coordsIn[6 * embedDim + 2] = R;
2593         coordsIn[7 * embedDim + 0] = R;
2594         coordsIn[7 * embedDim + 1] = -R;
2595         coordsIn[7 * embedDim + 2] = R;
2596       }
2597     }
2598     break;
2599   case 3:
2600     if (simplex) {
2601       const PetscReal edgeLen         = 1.0 / PETSC_PHI;
2602       PetscReal       vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
2603       PetscReal       vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
2604       PetscReal       vertexC[4]      = {0.5, 0.5 * PETSC_PHI, 0.5 / PETSC_PHI, 0.0};
2605       const PetscInt  degree          = 12;
2606       PetscInt        s[4]            = {1, 1, 1};
2607       PetscInt        evenPerm[12][4] = {
2608         {0, 1, 2, 3},
2609         {0, 2, 3, 1},
2610         {0, 3, 1, 2},
2611         {1, 0, 3, 2},
2612         {1, 2, 0, 3},
2613         {1, 3, 2, 0},
2614         {2, 0, 1, 3},
2615         {2, 1, 3, 0},
2616         {2, 3, 0, 1},
2617         {3, 0, 2, 1},
2618         {3, 1, 0, 2},
2619         {3, 2, 1, 0}
2620       };
2621       PetscInt  cone[4];
2622       PetscInt *graph, p, i, j, k, l;
2623 
2624       vertexA[0] *= R;
2625       vertexA[1] *= R;
2626       vertexA[2] *= R;
2627       vertexA[3] *= R;
2628       vertexB[0] *= R;
2629       vertexB[1] *= R;
2630       vertexB[2] *= R;
2631       vertexB[3] *= R;
2632       vertexC[0] *= R;
2633       vertexC[1] *= R;
2634       vertexC[2] *= R;
2635       vertexC[3] *= R;
2636       numCells    = rank == 0 ? 600 : 0;
2637       numVerts    = rank == 0 ? 120 : 0;
2638       firstVertex = numCells;
2639       /* Use the 600-cell, which for a unit sphere has coordinates which are
2640 
2641            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
2642                (\pm 1,    0,       0,      0)  all cyclic permutations   8
2643            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
2644 
2645          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2646          length is then given by 1/\phi = 0.61803.
2647 
2648          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
2649          http://mathworld.wolfram.com/600-Cell.html
2650       */
2651       /* Construct vertices */
2652       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2653       i = 0;
2654       if (rank == 0) {
2655         for (s[0] = -1; s[0] < 2; s[0] += 2) {
2656           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2657             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2658               for (s[3] = -1; s[3] < 2; s[3] += 2) {
2659                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[d] * vertexA[d];
2660                 ++i;
2661               }
2662             }
2663           }
2664         }
2665         for (p = 0; p < embedDim; ++p) {
2666           s[1] = s[2] = s[3] = 1;
2667           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2668             for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertexB[(d + p) % embedDim];
2669             ++i;
2670           }
2671         }
2672         for (p = 0; p < 12; ++p) {
2673           s[3] = 1;
2674           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2675             for (s[1] = -1; s[1] < 2; s[1] += 2) {
2676               for (s[2] = -1; s[2] < 2; s[2] += 2) {
2677                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[evenPerm[p][d]] * vertexC[evenPerm[p][d]];
2678                 ++i;
2679               }
2680             }
2681           }
2682         }
2683       }
2684       PetscCheck(i == numVerts, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertices %" PetscInt_FMT " != %" PetscInt_FMT, i, numVerts);
2685       /* Construct graph */
2686       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2687       for (i = 0; i < numVerts; ++i) {
2688         for (j = 0, k = 0; j < numVerts; ++j) {
2689           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2690             graph[i * numVerts + j] = 1;
2691             ++k;
2692           }
2693         }
2694         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2695       }
2696       /* Build Topology */
2697       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2698       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2699       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2700       /* Cells */
2701       if (rank == 0) {
2702         for (i = 0, c = 0; i < numVerts; ++i) {
2703           for (j = 0; j < i; ++j) {
2704             for (k = 0; k < j; ++k) {
2705               for (l = 0; l < k; ++l) {
2706                 if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i] && graph[l * numVerts + i] && graph[l * numVerts + j] && graph[l * numVerts + k]) {
2707                   cone[0] = firstVertex + i;
2708                   cone[1] = firstVertex + j;
2709                   cone[2] = firstVertex + k;
2710                   cone[3] = firstVertex + l;
2711                   /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
2712                   {
2713                     const PetscInt epsilon[4][4][4][4] = {
2714                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, -1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 0, 0}, {0, 1, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 1, 0}, {0, -1, 0, 0}, {0, 0, 0, 0}}},
2715 
2716                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}}, {{0, 0, -1, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}}},
2717 
2718                       {{{0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, 0, 0}, {0, -1, 0, 0}}, {{0, 0, 0, -1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 1, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}},
2719 
2720                       {{{0, 0, 0, 0}, {0, 0, -1, 0}, {0, 1, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 1, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}}, {{0, -1, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}} }
2721                     };
2722                     PetscReal normal[4];
2723                     PetscInt  e, f, g;
2724 
2725                     for (d = 0; d < embedDim; ++d) {
2726                       normal[d] = 0.0;
2727                       for (e = 0; e < embedDim; ++e) {
2728                         for (f = 0; f < embedDim; ++f) {
2729                           for (g = 0; g < embedDim; ++g) {
2730                             normal[d] += epsilon[d][e][f][g] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]) * (coordsIn[l * embedDim + f] - coordsIn[i * embedDim + f]);
2731                           }
2732                         }
2733                       }
2734                     }
2735                     if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2736                       PetscInt tmp = cone[1];
2737                       cone[1]      = cone[2];
2738                       cone[2]      = tmp;
2739                     }
2740                   }
2741                   PetscCall(DMPlexSetCone(dm, c++, cone));
2742                 }
2743               }
2744             }
2745           }
2746         }
2747       }
2748       PetscCall(DMPlexSymmetrize(dm));
2749       PetscCall(DMPlexStratify(dm));
2750       PetscCall(PetscFree(graph));
2751     }
2752     break;
2753   default:
2754     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unsupported dimension for sphere: %" PetscInt_FMT, dim);
2755   }
2756   /* Create coordinates */
2757   PetscCall(DMGetCoordinateSection(dm, &coordSection));
2758   PetscCall(PetscSectionSetNumFields(coordSection, 1));
2759   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, embedDim));
2760   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVerts));
2761   for (v = firstVertex; v < firstVertex + numVerts; ++v) {
2762     PetscCall(PetscSectionSetDof(coordSection, v, embedDim));
2763     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, embedDim));
2764   }
2765   PetscCall(PetscSectionSetUp(coordSection));
2766   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2767   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2768   PetscCall(VecSetBlockSize(coordinates, embedDim));
2769   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2770   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2771   PetscCall(VecSetType(coordinates, VECSTANDARD));
2772   PetscCall(VecGetArray(coordinates, &coords));
2773   for (v = 0; v < numVerts; ++v)
2774     for (d = 0; d < embedDim; ++d) coords[v * embedDim + d] = coordsIn[v * embedDim + d];
2775   PetscCall(VecRestoreArray(coordinates, &coords));
2776   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2777   PetscCall(VecDestroy(&coordinates));
2778   PetscCall(PetscFree(coordsIn));
2779   {
2780     DM          cdm;
2781     PetscDS     cds;
2782     PetscScalar c = R;
2783 
2784     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToSphere));
2785     PetscCall(DMGetCoordinateDM(dm, &cdm));
2786     PetscCall(DMGetDS(cdm, &cds));
2787     PetscCall(PetscDSSetConstants(cds, 1, &c));
2788   }
2789   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
2790   /* Wait for coordinate creation before doing in-place modification */
2791   if (simplex) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2792   PetscFunctionReturn(PETSC_SUCCESS);
2793 }
2794 
2795 typedef void (*TPSEvaluateFunc)(const PetscReal[], PetscReal *, PetscReal[], PetscReal (*)[3]);
2796 
2797 /*
2798  The Schwarz P implicit surface is
2799 
2800      f(x) = cos(x0) + cos(x1) + cos(x2) = 0
2801 */
2802 static void TPSEvaluate_SchwarzP(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2803 {
2804   PetscReal c[3] = {PetscCosReal(y[0] * PETSC_PI), PetscCosReal(y[1] * PETSC_PI), PetscCosReal(y[2] * PETSC_PI)};
2805   PetscReal g[3] = {-PetscSinReal(y[0] * PETSC_PI), -PetscSinReal(y[1] * PETSC_PI), -PetscSinReal(y[2] * PETSC_PI)};
2806   f[0]           = c[0] + c[1] + c[2];
2807   for (PetscInt i = 0; i < 3; i++) {
2808     grad[i] = PETSC_PI * g[i];
2809     for (PetscInt j = 0; j < 3; j++) hess[i][j] = (i == j) ? -PetscSqr(PETSC_PI) * c[i] : 0.;
2810   }
2811 }
2812 
2813 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2814 static PetscErrorCode TPSExtrudeNormalFunc_SchwarzP(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2815 {
2816   for (PetscInt i = 0; i < 3; i++) u[i] = -PETSC_PI * PetscSinReal(x[i] * PETSC_PI);
2817   return PETSC_SUCCESS;
2818 }
2819 
2820 /*
2821  The Gyroid implicit surface is
2822 
2823  f(x,y,z) = sin(pi * x) * cos (pi * (y + 1/2))  + sin(pi * (y + 1/2)) * cos(pi * (z + 1/4)) + sin(pi * (z + 1/4)) * cos(pi * x)
2824 
2825 */
2826 static void TPSEvaluate_Gyroid(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2827 {
2828   PetscReal s[3] = {PetscSinReal(PETSC_PI * y[0]), PetscSinReal(PETSC_PI * (y[1] + .5)), PetscSinReal(PETSC_PI * (y[2] + .25))};
2829   PetscReal c[3] = {PetscCosReal(PETSC_PI * y[0]), PetscCosReal(PETSC_PI * (y[1] + .5)), PetscCosReal(PETSC_PI * (y[2] + .25))};
2830   f[0]           = s[0] * c[1] + s[1] * c[2] + s[2] * c[0];
2831   grad[0]        = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2832   grad[1]        = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2833   grad[2]        = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2834   hess[0][0]     = -PetscSqr(PETSC_PI) * (s[0] * c[1] + s[2] * c[0]);
2835   hess[0][1]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2836   hess[0][2]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2837   hess[1][0]     = -PetscSqr(PETSC_PI) * (s[1] * c[2] + s[0] * c[1]);
2838   hess[1][1]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2839   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2840   hess[2][0]     = -PetscSqr(PETSC_PI) * (s[2] * c[0] + s[1] * c[2]);
2841   hess[2][1]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2842   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2843 }
2844 
2845 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2846 static PetscErrorCode TPSExtrudeNormalFunc_Gyroid(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2847 {
2848   PetscReal s[3] = {PetscSinReal(PETSC_PI * x[0]), PetscSinReal(PETSC_PI * (x[1] + .5)), PetscSinReal(PETSC_PI * (x[2] + .25))};
2849   PetscReal c[3] = {PetscCosReal(PETSC_PI * x[0]), PetscCosReal(PETSC_PI * (x[1] + .5)), PetscCosReal(PETSC_PI * (x[2] + .25))};
2850   u[0]           = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2851   u[1]           = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2852   u[2]           = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2853   return PETSC_SUCCESS;
2854 }
2855 
2856 /*
2857    We wish to solve
2858 
2859          min_y || y - x ||^2  subject to f(y) = 0
2860 
2861    Let g(y) = grad(f).  The minimization problem is equivalent to asking to satisfy
2862    f(y) = 0 and (y-x) is parallel to g(y).  We do this by using Householder QR to obtain a basis for the
2863    tangent space and ask for both components in the tangent space to be zero.
2864 
2865    Take g to be a column vector and compute the "full QR" factorization Q R = g,
2866    where Q = I - 2 n n^T is a symmetric orthogonal matrix.
2867    The first column of Q is parallel to g so the remaining two columns span the null space.
2868    Let Qn = Q[:,1:] be those remaining columns.  Then Qn Qn^T is an orthogonal projector into the tangent space.
2869    Since Q is symmetric, this is equivalent to multiplying by Q and taking the last two entries.
2870    In total, we have a system of 3 equations in 3 unknowns:
2871 
2872      f(y) = 0                       1 equation
2873      Qn^T (y - x) = 0               2 equations
2874 
2875    Here, we compute the residual and Jacobian of this system.
2876 */
2877 static void TPSNearestPointResJac(TPSEvaluateFunc feval, const PetscScalar x[], const PetscScalar y[], PetscScalar res[], PetscScalar J[])
2878 {
2879   PetscReal yreal[3] = {PetscRealPart(y[0]), PetscRealPart(y[1]), PetscRealPart(y[2])};
2880   PetscReal d[3]     = {PetscRealPart(y[0] - x[0]), PetscRealPart(y[1] - x[1]), PetscRealPart(y[2] - x[2])};
2881   PetscReal f, grad[3], n[3], norm, norm_y[3], nd, nd_y[3], sign;
2882   PetscReal n_y[3][3] = {
2883     {0, 0, 0},
2884     {0, 0, 0},
2885     {0, 0, 0}
2886   };
2887 
2888   feval(yreal, &f, grad, n_y);
2889 
2890   for (PetscInt i = 0; i < 3; i++) n[i] = grad[i];
2891   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2892   for (PetscInt i = 0; i < 3; i++) norm_y[i] = 1. / norm * n[i] * n_y[i][i];
2893 
2894   // Define the Householder reflector
2895   sign = n[0] >= 0 ? 1. : -1.;
2896   n[0] += norm * sign;
2897   for (PetscInt i = 0; i < 3; i++) n_y[0][i] += norm_y[i] * sign;
2898 
2899   norm      = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2900   norm_y[0] = 1. / norm * (n[0] * n_y[0][0]);
2901   norm_y[1] = 1. / norm * (n[0] * n_y[0][1] + n[1] * n_y[1][1]);
2902   norm_y[2] = 1. / norm * (n[0] * n_y[0][2] + n[2] * n_y[2][2]);
2903 
2904   for (PetscInt i = 0; i < 3; i++) {
2905     n[i] /= norm;
2906     for (PetscInt j = 0; j < 3; j++) {
2907       // note that n[i] is n_old[i]/norm when executing the code below
2908       n_y[i][j] = n_y[i][j] / norm - n[i] / norm * norm_y[j];
2909     }
2910   }
2911 
2912   nd = n[0] * d[0] + n[1] * d[1] + n[2] * d[2];
2913   for (PetscInt i = 0; i < 3; i++) nd_y[i] = n[i] + n_y[0][i] * d[0] + n_y[1][i] * d[1] + n_y[2][i] * d[2];
2914 
2915   res[0] = f;
2916   res[1] = d[1] - 2 * n[1] * nd;
2917   res[2] = d[2] - 2 * n[2] * nd;
2918   // J[j][i] is J_{ij} (column major)
2919   for (PetscInt j = 0; j < 3; j++) {
2920     J[0 + j * 3] = grad[j];
2921     J[1 + j * 3] = (j == 1) * 1. - 2 * (n_y[1][j] * nd + n[1] * nd_y[j]);
2922     J[2 + j * 3] = (j == 2) * 1. - 2 * (n_y[2][j] * nd + n[2] * nd_y[j]);
2923   }
2924 }
2925 
2926 /*
2927    Project x to the nearest point on the implicit surface using Newton's method.
2928 */
2929 static PetscErrorCode TPSNearestPoint(TPSEvaluateFunc feval, PetscScalar x[])
2930 {
2931   PetscScalar y[3] = {x[0], x[1], x[2]}; // Initial guess
2932 
2933   PetscFunctionBegin;
2934   for (PetscInt iter = 0; iter < 10; iter++) {
2935     PetscScalar res[3], J[9];
2936     PetscReal   resnorm;
2937     TPSNearestPointResJac(feval, x, y, res, J);
2938     resnorm = PetscSqrtReal(PetscSqr(PetscRealPart(res[0])) + PetscSqr(PetscRealPart(res[1])) + PetscSqr(PetscRealPart(res[2])));
2939     if (0) { // Turn on this monitor if you need to confirm quadratic convergence
2940       PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%" PetscInt_FMT "] res [%g %g %g]\n", iter, (double)PetscRealPart(res[0]), (double)PetscRealPart(res[1]), (double)PetscRealPart(res[2])));
2941     }
2942     if (resnorm < PETSC_SMALL) break;
2943 
2944     // Take the Newton step
2945     PetscCall(PetscKernel_A_gets_inverse_A_3(J, 0., PETSC_FALSE, NULL));
2946     PetscKernel_v_gets_v_minus_A_times_w_3(y, J, res);
2947   }
2948   for (PetscInt i = 0; i < 3; i++) x[i] = y[i];
2949   PetscFunctionReturn(PETSC_SUCCESS);
2950 }
2951 
2952 const char *const DMPlexTPSTypes[] = {"SCHWARZ_P", "GYROID", "DMPlexTPSType", "DMPLEX_TPS_", NULL};
2953 
2954 static PetscErrorCode DMPlexCreateTPSMesh_Internal(DM dm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness)
2955 {
2956   PetscMPIInt rank;
2957   PetscInt    topoDim = 2, spaceDim = 3, numFaces = 0, numVertices = 0, numEdges = 0;
2958   PetscInt(*edges)[2] = NULL, *edgeSets = NULL;
2959   PetscInt            *cells_flat = NULL;
2960   PetscReal           *vtxCoords  = NULL;
2961   TPSEvaluateFunc      evalFunc   = NULL;
2962   PetscSimplePointFunc normalFunc = NULL;
2963   DMLabel              label;
2964 
2965   PetscFunctionBegin;
2966   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
2967   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2968   PetscCheck((layers != 0) ^ (thickness == 0.), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Layers %" PetscInt_FMT " must be nonzero iff thickness %g is nonzero", layers, (double)thickness);
2969   switch (tpstype) {
2970   case DMPLEX_TPS_SCHWARZ_P:
2971     PetscCheck(!periodic || (periodic[0] == DM_BOUNDARY_NONE && periodic[1] == DM_BOUNDARY_NONE && periodic[2] == DM_BOUNDARY_NONE), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Schwarz P does not support periodic meshes");
2972     if (rank == 0) {
2973       PetscInt(*cells)[6][4][4] = NULL; // [junction, junction-face, cell, conn]
2974       PetscInt  Njunctions = 0, Ncuts = 0, Npipes[3], vcount;
2975       PetscReal L = 1;
2976 
2977       Npipes[0]   = (extent[0] + 1) * extent[1] * extent[2];
2978       Npipes[1]   = extent[0] * (extent[1] + 1) * extent[2];
2979       Npipes[2]   = extent[0] * extent[1] * (extent[2] + 1);
2980       Njunctions  = extent[0] * extent[1] * extent[2];
2981       Ncuts       = 2 * (extent[0] * extent[1] + extent[1] * extent[2] + extent[2] * extent[0]);
2982       numVertices = 4 * (Npipes[0] + Npipes[1] + Npipes[2]) + 8 * Njunctions;
2983       PetscCall(PetscMalloc1(3 * numVertices, &vtxCoords));
2984       PetscCall(PetscMalloc1(Njunctions, &cells));
2985       PetscCall(PetscMalloc1(Ncuts * 4, &edges));
2986       PetscCall(PetscMalloc1(Ncuts * 4, &edgeSets));
2987       // x-normal pipes
2988       vcount = 0;
2989       for (PetscInt i = 0; i < extent[0] + 1; i++) {
2990         for (PetscInt j = 0; j < extent[1]; j++) {
2991           for (PetscInt k = 0; k < extent[2]; k++) {
2992             for (PetscInt l = 0; l < 4; l++) {
2993               vtxCoords[vcount++] = (2 * i - 1) * L;
2994               vtxCoords[vcount++] = 2 * j * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2995               vtxCoords[vcount++] = 2 * k * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2996             }
2997           }
2998         }
2999       }
3000       // y-normal pipes
3001       for (PetscInt i = 0; i < extent[0]; i++) {
3002         for (PetscInt j = 0; j < extent[1] + 1; j++) {
3003           for (PetscInt k = 0; k < extent[2]; k++) {
3004             for (PetscInt l = 0; l < 4; l++) {
3005               vtxCoords[vcount++] = 2 * i * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3006               vtxCoords[vcount++] = (2 * j - 1) * L;
3007               vtxCoords[vcount++] = 2 * k * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3008             }
3009           }
3010         }
3011       }
3012       // z-normal pipes
3013       for (PetscInt i = 0; i < extent[0]; i++) {
3014         for (PetscInt j = 0; j < extent[1]; j++) {
3015           for (PetscInt k = 0; k < extent[2] + 1; k++) {
3016             for (PetscInt l = 0; l < 4; l++) {
3017               vtxCoords[vcount++] = 2 * i * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3018               vtxCoords[vcount++] = 2 * j * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3019               vtxCoords[vcount++] = (2 * k - 1) * L;
3020             }
3021           }
3022         }
3023       }
3024       // junctions
3025       for (PetscInt i = 0; i < extent[0]; i++) {
3026         for (PetscInt j = 0; j < extent[1]; j++) {
3027           for (PetscInt k = 0; k < extent[2]; k++) {
3028             const PetscInt J = (i * extent[1] + j) * extent[2] + k, Jvoff = (Npipes[0] + Npipes[1] + Npipes[2]) * 4 + J * 8;
3029             PetscCheck(vcount / 3 == Jvoff, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected vertex count");
3030             for (PetscInt ii = 0; ii < 2; ii++) {
3031               for (PetscInt jj = 0; jj < 2; jj++) {
3032                 for (PetscInt kk = 0; kk < 2; kk++) {
3033                   double Ls           = (1 - sqrt(2) / 4) * L;
3034                   vtxCoords[vcount++] = 2 * i * L + (2 * ii - 1) * Ls;
3035                   vtxCoords[vcount++] = 2 * j * L + (2 * jj - 1) * Ls;
3036                   vtxCoords[vcount++] = 2 * k * L + (2 * kk - 1) * Ls;
3037                 }
3038               }
3039             }
3040             const PetscInt jfaces[3][2][4] = {
3041               {{3, 1, 0, 2}, {7, 5, 4, 6}}, // x-aligned
3042               {{5, 4, 0, 1}, {7, 6, 2, 3}}, // y-aligned
3043               {{6, 2, 0, 4}, {7, 3, 1, 5}}  // z-aligned
3044             };
3045             const PetscInt pipe_lo[3] = {// vertex numbers of pipes
3046                                          ((i * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + Npipes[0] + Npipes[1]) * 4};
3047             const PetscInt pipe_hi[3] = {// vertex numbers of pipes
3048                                          (((i + 1) * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j + 1) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + 1 + Npipes[0] + Npipes[1]) * 4};
3049             for (PetscInt dir = 0; dir < 3; dir++) { // x,y,z
3050               const PetscInt ijk[3] = {i, j, k};
3051               for (PetscInt l = 0; l < 4; l++) { // rotations
3052                 cells[J][dir * 2 + 0][l][0] = pipe_lo[dir] + l;
3053                 cells[J][dir * 2 + 0][l][1] = Jvoff + jfaces[dir][0][l];
3054                 cells[J][dir * 2 + 0][l][2] = Jvoff + jfaces[dir][0][(l - 1 + 4) % 4];
3055                 cells[J][dir * 2 + 0][l][3] = pipe_lo[dir] + (l - 1 + 4) % 4;
3056                 cells[J][dir * 2 + 1][l][0] = Jvoff + jfaces[dir][1][l];
3057                 cells[J][dir * 2 + 1][l][1] = pipe_hi[dir] + l;
3058                 cells[J][dir * 2 + 1][l][2] = pipe_hi[dir] + (l - 1 + 4) % 4;
3059                 cells[J][dir * 2 + 1][l][3] = Jvoff + jfaces[dir][1][(l - 1 + 4) % 4];
3060                 if (ijk[dir] == 0) {
3061                   edges[numEdges][0] = pipe_lo[dir] + l;
3062                   edges[numEdges][1] = pipe_lo[dir] + (l + 1) % 4;
3063                   edgeSets[numEdges] = dir * 2 + 1;
3064                   numEdges++;
3065                 }
3066                 if (ijk[dir] + 1 == extent[dir]) {
3067                   edges[numEdges][0] = pipe_hi[dir] + l;
3068                   edges[numEdges][1] = pipe_hi[dir] + (l + 1) % 4;
3069                   edgeSets[numEdges] = dir * 2 + 2;
3070                   numEdges++;
3071                 }
3072               }
3073             }
3074           }
3075         }
3076       }
3077       PetscCheck(numEdges == Ncuts * 4, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Edge count %" PetscInt_FMT " incompatible with number of cuts %" PetscInt_FMT, numEdges, Ncuts);
3078       numFaces   = 24 * Njunctions;
3079       cells_flat = cells[0][0][0];
3080     }
3081     evalFunc   = TPSEvaluate_SchwarzP;
3082     normalFunc = TPSExtrudeNormalFunc_SchwarzP;
3083     break;
3084   case DMPLEX_TPS_GYROID:
3085     if (rank == 0) {
3086       // This is a coarse mesh approximation of the gyroid shifted to being the zero of the level set
3087       //
3088       //     sin(pi*x)*cos(pi*(y+1/2)) + sin(pi*(y+1/2))*cos(pi*(z+1/4)) + sin(pi*(z+1/4))*cos(x)
3089       //
3090       // on the cell [0,2]^3.
3091       //
3092       // Think about dividing that cell into four columns, and focus on the column [0,1]x[0,1]x[0,2].
3093       // If you looked at the gyroid in that column at different slices of z you would see that it kind of spins
3094       // like a boomerang:
3095       //
3096       //     z = 0          z = 1/4        z = 1/2        z = 3/4     //
3097       //     -----          -------        -------        -------     //
3098       //                                                              //
3099       //     +       +      +       +      +       +      +   \   +   //
3100       //      \                                   /            \      //
3101       //       \            `-_   _-'            /              }     //
3102       //        *-_            `-'            _-'              /      //
3103       //     +     `-+      +       +      +-'     +      +   /   +   //
3104       //                                                              //
3105       //                                                              //
3106       //     z = 1          z = 5/4        z = 3/2        z = 7/4     //
3107       //     -----          -------        -------        -------     //
3108       //                                                              //
3109       //     +-_     +      +       +      +     _-+      +   /   +   //
3110       //        `-_            _-_            _-`            /        //
3111       //           \        _-'   `-_        /              {         //
3112       //            \                       /                \        //
3113       //     +       +      +       +      +       +      +   \   +   //
3114       //
3115       //
3116       // This course mesh approximates each of these slices by two line segments,
3117       // and then connects the segments in consecutive layers with quadrilateral faces.
3118       // All of the end points of the segments are multiples of 1/4 except for the
3119       // point * in the picture for z = 0 above and the similar points in other layers.
3120       // That point is at (gamma, gamma, 0), where gamma is calculated below.
3121       //
3122       // The column  [1,2]x[1,2]x[0,2] looks the same as this column;
3123       // The columns [1,2]x[0,1]x[0,2] and [0,1]x[1,2]x[0,2] are mirror images.
3124       //
3125       // As for how this method turned into the names given to the vertices:
3126       // that was not systematic, it was just the way it worked out in my handwritten notes.
3127 
3128       PetscInt facesPerBlock = 64;
3129       PetscInt vertsPerBlock = 56;
3130       PetscInt extentPlus[3];
3131       PetscInt numBlocks, numBlocksPlus;
3132       const PetscInt A = 0, B = 1, C = 2, D = 3, E = 4, F = 5, G = 6, H = 7, II = 8, J = 9, K = 10, L = 11, M = 12, N = 13, O = 14, P = 15, Q = 16, R = 17, S = 18, T = 19, U = 20, V = 21, W = 22, X = 23, Y = 24, Z = 25, Ap = 26, Bp = 27, Cp = 28, Dp = 29, Ep = 30, Fp = 31, Gp = 32, Hp = 33, Ip = 34, Jp = 35, Kp = 36, Lp = 37, Mp = 38, Np = 39, Op = 40, Pp = 41, Qp = 42, Rp = 43, Sp = 44, Tp = 45, Up = 46, Vp = 47, Wp = 48, Xp = 49, Yp = 50, Zp = 51, Aq = 52, Bq = 53, Cq = 54, Dq = 55;
3133       const PetscInt pattern[64][4] = {
3134   /* face to vertex within the coarse discretization of a single gyroid block */
3135   /* layer 0 */
3136         {A,           C,           K,           G          },
3137         {C,           B,           II,          K          },
3138         {D,           A,           H,           L          },
3139         {B + 56 * 1,  D,           L,           J          },
3140         {E,           B + 56 * 1,  J,           N          },
3141         {A + 56 * 2,  E,           N,           H + 56 * 2 },
3142         {F,           A + 56 * 2,  G + 56 * 2,  M          },
3143         {B,           F,           M,           II         },
3144  /* layer 1 */
3145         {G,           K,           Q,           O          },
3146         {K,           II,          P,           Q          },
3147         {L,           H,           O + 56 * 1,  R          },
3148         {J,           L,           R,           P          },
3149         {N,           J,           P,           S          },
3150         {H + 56 * 2,  N,           S,           O + 56 * 3 },
3151         {M,           G + 56 * 2,  O + 56 * 2,  T          },
3152         {II,          M,           T,           P          },
3153  /* layer 2 */
3154         {O,           Q,           Y,           U          },
3155         {Q,           P,           W,           Y          },
3156         {R,           O + 56 * 1,  U + 56 * 1,  Ap         },
3157         {P,           R,           Ap,          W          },
3158         {S,           P,           X,           Bp         },
3159         {O + 56 * 3,  S,           Bp,          V + 56 * 1 },
3160         {T,           O + 56 * 2,  V,           Z          },
3161         {P,           T,           Z,           X          },
3162  /* layer 3 */
3163         {U,           Y,           Ep,          Dp         },
3164         {Y,           W,           Cp,          Ep         },
3165         {Ap,          U + 56 * 1,  Dp + 56 * 1, Gp         },
3166         {W,           Ap,          Gp,          Cp         },
3167         {Bp,          X,           Cp + 56 * 2, Fp         },
3168         {V + 56 * 1,  Bp,          Fp,          Dp + 56 * 1},
3169         {Z,           V,           Dp,          Hp         },
3170         {X,           Z,           Hp,          Cp + 56 * 2},
3171  /* layer 4 */
3172         {Dp,          Ep,          Mp,          Kp         },
3173         {Ep,          Cp,          Ip,          Mp         },
3174         {Gp,          Dp + 56 * 1, Lp,          Np         },
3175         {Cp,          Gp,          Np,          Jp         },
3176         {Fp,          Cp + 56 * 2, Jp + 56 * 2, Pp         },
3177         {Dp + 56 * 1, Fp,          Pp,          Lp         },
3178         {Hp,          Dp,          Kp,          Op         },
3179         {Cp + 56 * 2, Hp,          Op,          Ip + 56 * 2},
3180  /* layer 5 */
3181         {Kp,          Mp,          Sp,          Rp         },
3182         {Mp,          Ip,          Qp,          Sp         },
3183         {Np,          Lp,          Rp,          Tp         },
3184         {Jp,          Np,          Tp,          Qp + 56 * 1},
3185         {Pp,          Jp + 56 * 2, Qp + 56 * 3, Up         },
3186         {Lp,          Pp,          Up,          Rp         },
3187         {Op,          Kp,          Rp,          Vp         },
3188         {Ip + 56 * 2, Op,          Vp,          Qp + 56 * 2},
3189  /* layer 6 */
3190         {Rp,          Sp,          Aq,          Yp         },
3191         {Sp,          Qp,          Wp,          Aq         },
3192         {Tp,          Rp,          Yp,          Cq         },
3193         {Qp + 56 * 1, Tp,          Cq,          Wp + 56 * 1},
3194         {Up,          Qp + 56 * 3, Xp + 56 * 1, Dq         },
3195         {Rp,          Up,          Dq,          Zp         },
3196         {Vp,          Rp,          Zp,          Bq         },
3197         {Qp + 56 * 2, Vp,          Bq,          Xp         },
3198  /* layer 7 (the top is the periodic image of the bottom of layer 0) */
3199         {Yp,          Aq,          C + 56 * 4,  A + 56 * 4 },
3200         {Aq,          Wp,          B + 56 * 4,  C + 56 * 4 },
3201         {Cq,          Yp,          A + 56 * 4,  D + 56 * 4 },
3202         {Wp + 56 * 1, Cq,          D + 56 * 4,  B + 56 * 5 },
3203         {Dq,          Xp + 56 * 1, B + 56 * 5,  E + 56 * 4 },
3204         {Zp,          Dq,          E + 56 * 4,  A + 56 * 6 },
3205         {Bq,          Zp,          A + 56 * 6,  F + 56 * 4 },
3206         {Xp,          Bq,          F + 56 * 4,  B + 56 * 4 }
3207       };
3208       const PetscReal gamma                = PetscAcosReal((PetscSqrtReal(3.) - 1.) / PetscSqrtReal(2.)) / PETSC_PI;
3209       const PetscReal patternCoords[56][3] = {
3210         {1.,        0.,        0.  }, /* A  */
3211         {0.,        1.,        0.  }, /* B  */
3212         {gamma,     gamma,     0.  }, /* C  */
3213         {1 + gamma, 1 - gamma, 0.  }, /* D  */
3214         {2 - gamma, 2 - gamma, 0.  }, /* E  */
3215         {1 - gamma, 1 + gamma, 0.  }, /* F  */
3216 
3217         {.5,        0,         .25 }, /* G  */
3218         {1.5,       0.,        .25 }, /* H  */
3219         {.5,        1.,        .25 }, /* II */
3220         {1.5,       1.,        .25 }, /* J  */
3221         {.25,       .5,        .25 }, /* K  */
3222         {1.25,      .5,        .25 }, /* L  */
3223         {.75,       1.5,       .25 }, /* M  */
3224         {1.75,      1.5,       .25 }, /* N  */
3225 
3226         {0.,        0.,        .5  }, /* O  */
3227         {1.,        1.,        .5  }, /* P  */
3228         {gamma,     1 - gamma, .5  }, /* Q  */
3229         {1 + gamma, gamma,     .5  }, /* R  */
3230         {2 - gamma, 1 + gamma, .5  }, /* S  */
3231         {1 - gamma, 2 - gamma, .5  }, /* T  */
3232 
3233         {0.,        .5,        .75 }, /* U  */
3234         {0.,        1.5,       .75 }, /* V  */
3235         {1.,        .5,        .75 }, /* W  */
3236         {1.,        1.5,       .75 }, /* X  */
3237         {.5,        .75,       .75 }, /* Y  */
3238         {.5,        1.75,      .75 }, /* Z  */
3239         {1.5,       .25,       .75 }, /* Ap */
3240         {1.5,       1.25,      .75 }, /* Bp */
3241 
3242         {1.,        0.,        1.  }, /* Cp */
3243         {0.,        1.,        1.  }, /* Dp */
3244         {1 - gamma, 1 - gamma, 1.  }, /* Ep */
3245         {1 + gamma, 1 + gamma, 1.  }, /* Fp */
3246         {2 - gamma, gamma,     1.  }, /* Gp */
3247         {gamma,     2 - gamma, 1.  }, /* Hp */
3248 
3249         {.5,        0.,        1.25}, /* Ip */
3250         {1.5,       0.,        1.25}, /* Jp */
3251         {.5,        1.,        1.25}, /* Kp */
3252         {1.5,       1.,        1.25}, /* Lp */
3253         {.75,       .5,        1.25}, /* Mp */
3254         {1.75,      .5,        1.25}, /* Np */
3255         {.25,       1.5,       1.25}, /* Op */
3256         {1.25,      1.5,       1.25}, /* Pp */
3257 
3258         {0.,        0.,        1.5 }, /* Qp */
3259         {1.,        1.,        1.5 }, /* Rp */
3260         {1 - gamma, gamma,     1.5 }, /* Sp */
3261         {2 - gamma, 1 - gamma, 1.5 }, /* Tp */
3262         {1 + gamma, 2 - gamma, 1.5 }, /* Up */
3263         {gamma,     1 + gamma, 1.5 }, /* Vp */
3264 
3265         {0.,        .5,        1.75}, /* Wp */
3266         {0.,        1.5,       1.75}, /* Xp */
3267         {1.,        .5,        1.75}, /* Yp */
3268         {1.,        1.5,       1.75}, /* Zp */
3269         {.5,        .25,       1.75}, /* Aq */
3270         {.5,        1.25,      1.75}, /* Bq */
3271         {1.5,       .75,       1.75}, /* Cq */
3272         {1.5,       1.75,      1.75}, /* Dq */
3273       };
3274       PetscInt(*cells)[64][4] = NULL;
3275       PetscBool *seen;
3276       PetscInt  *vertToTrueVert;
3277       PetscInt   count;
3278 
3279       for (PetscInt i = 0; i < 3; i++) extentPlus[i] = extent[i] + 1;
3280       numBlocks = 1;
3281       for (PetscInt i = 0; i < 3; i++) numBlocks *= extent[i];
3282       numBlocksPlus = 1;
3283       for (PetscInt i = 0; i < 3; i++) numBlocksPlus *= extentPlus[i];
3284       numFaces = numBlocks * facesPerBlock;
3285       PetscCall(PetscMalloc1(numBlocks, &cells));
3286       PetscCall(PetscCalloc1(numBlocksPlus * vertsPerBlock, &seen));
3287       for (PetscInt k = 0; k < extent[2]; k++) {
3288         for (PetscInt j = 0; j < extent[1]; j++) {
3289           for (PetscInt i = 0; i < extent[0]; i++) {
3290             for (PetscInt f = 0; f < facesPerBlock; f++) {
3291               for (PetscInt v = 0; v < 4; v++) {
3292                 PetscInt vertRaw     = pattern[f][v];
3293                 PetscInt blockidx    = vertRaw / 56;
3294                 PetscInt patternvert = vertRaw % 56;
3295                 PetscInt xplus       = (blockidx & 1);
3296                 PetscInt yplus       = (blockidx & 2) >> 1;
3297                 PetscInt zplus       = (blockidx & 4) >> 2;
3298                 PetscInt zcoord      = (periodic && periodic[2] == DM_BOUNDARY_PERIODIC) ? ((k + zplus) % extent[2]) : (k + zplus);
3299                 PetscInt ycoord      = (periodic && periodic[1] == DM_BOUNDARY_PERIODIC) ? ((j + yplus) % extent[1]) : (j + yplus);
3300                 PetscInt xcoord      = (periodic && periodic[0] == DM_BOUNDARY_PERIODIC) ? ((i + xplus) % extent[0]) : (i + xplus);
3301                 PetscInt vert        = ((zcoord * extentPlus[1] + ycoord) * extentPlus[0] + xcoord) * 56 + patternvert;
3302 
3303                 cells[(k * extent[1] + j) * extent[0] + i][f][v] = vert;
3304                 seen[vert]                                       = PETSC_TRUE;
3305               }
3306             }
3307           }
3308         }
3309       }
3310       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++)
3311         if (seen[i]) numVertices++;
3312       count = 0;
3313       PetscCall(PetscMalloc1(numBlocksPlus * vertsPerBlock, &vertToTrueVert));
3314       PetscCall(PetscMalloc1(numVertices * 3, &vtxCoords));
3315       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) vertToTrueVert[i] = -1;
3316       for (PetscInt k = 0; k < extentPlus[2]; k++) {
3317         for (PetscInt j = 0; j < extentPlus[1]; j++) {
3318           for (PetscInt i = 0; i < extentPlus[0]; i++) {
3319             for (PetscInt v = 0; v < vertsPerBlock; v++) {
3320               PetscInt vIdx = ((k * extentPlus[1] + j) * extentPlus[0] + i) * vertsPerBlock + v;
3321 
3322               if (seen[vIdx]) {
3323                 PetscInt thisVert;
3324 
3325                 vertToTrueVert[vIdx] = thisVert = count++;
3326 
3327                 for (PetscInt d = 0; d < 3; d++) vtxCoords[3 * thisVert + d] = patternCoords[v][d];
3328                 vtxCoords[3 * thisVert + 0] += i * 2;
3329                 vtxCoords[3 * thisVert + 1] += j * 2;
3330                 vtxCoords[3 * thisVert + 2] += k * 2;
3331               }
3332             }
3333           }
3334         }
3335       }
3336       for (PetscInt i = 0; i < numBlocks; i++) {
3337         for (PetscInt f = 0; f < facesPerBlock; f++) {
3338           for (PetscInt v = 0; v < 4; v++) cells[i][f][v] = vertToTrueVert[cells[i][f][v]];
3339         }
3340       }
3341       PetscCall(PetscFree(vertToTrueVert));
3342       PetscCall(PetscFree(seen));
3343       cells_flat = cells[0][0];
3344       numEdges   = 0;
3345       for (PetscInt i = 0; i < numFaces; i++) {
3346         for (PetscInt e = 0; e < 4; e++) {
3347           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3348           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3349 
3350           for (PetscInt d = 0; d < 3; d++) {
3351             if (!periodic || periodic[0] != DM_BOUNDARY_PERIODIC) {
3352               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) numEdges++;
3353               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) numEdges++;
3354             }
3355           }
3356         }
3357       }
3358       PetscCall(PetscMalloc1(numEdges, &edges));
3359       PetscCall(PetscMalloc1(numEdges, &edgeSets));
3360       for (PetscInt edge = 0, i = 0; i < numFaces; i++) {
3361         for (PetscInt e = 0; e < 4; e++) {
3362           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3363           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3364 
3365           for (PetscInt d = 0; d < 3; d++) {
3366             if (!periodic || periodic[d] != DM_BOUNDARY_PERIODIC) {
3367               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) {
3368                 edges[edge][0]   = ev[0];
3369                 edges[edge][1]   = ev[1];
3370                 edgeSets[edge++] = 2 * d;
3371               }
3372               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) {
3373                 edges[edge][0]   = ev[0];
3374                 edges[edge][1]   = ev[1];
3375                 edgeSets[edge++] = 2 * d + 1;
3376               }
3377             }
3378           }
3379         }
3380       }
3381     }
3382     evalFunc   = TPSEvaluate_Gyroid;
3383     normalFunc = TPSExtrudeNormalFunc_Gyroid;
3384     break;
3385   }
3386 
3387   PetscCall(DMSetDimension(dm, topoDim));
3388   if (rank == 0) PetscCall(DMPlexBuildFromCellList(dm, numFaces, numVertices, 4, cells_flat));
3389   else PetscCall(DMPlexBuildFromCellList(dm, 0, 0, 0, NULL));
3390   PetscCall(PetscFree(cells_flat));
3391   {
3392     DM idm;
3393     PetscCall(DMPlexInterpolate(dm, &idm));
3394     PetscCall(DMPlexReplace_Internal(dm, &idm));
3395   }
3396   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, vtxCoords));
3397   else PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, NULL));
3398   PetscCall(PetscFree(vtxCoords));
3399 
3400   PetscCall(DMCreateLabel(dm, "Face Sets"));
3401   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3402   for (PetscInt e = 0; e < numEdges; e++) {
3403     PetscInt        njoin;
3404     const PetscInt *join, verts[] = {numFaces + edges[e][0], numFaces + edges[e][1]};
3405     PetscCall(DMPlexGetJoin(dm, 2, verts, &njoin, &join));
3406     PetscCheck(njoin == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected unique join of vertices %" PetscInt_FMT " and %" PetscInt_FMT, edges[e][0], edges[e][1]);
3407     PetscCall(DMLabelSetValue(label, join[0], edgeSets[e]));
3408     PetscCall(DMPlexRestoreJoin(dm, 2, verts, &njoin, &join));
3409   }
3410   PetscCall(PetscFree(edges));
3411   PetscCall(PetscFree(edgeSets));
3412   if (tps_distribute) {
3413     DM               pdm = NULL;
3414     PetscPartitioner part;
3415 
3416     PetscCall(DMPlexGetPartitioner(dm, &part));
3417     PetscCall(PetscPartitionerSetFromOptions(part));
3418     PetscCall(DMPlexDistribute(dm, 0, NULL, &pdm));
3419     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
3420     // Do not auto-distribute again
3421     PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
3422   }
3423 
3424   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3425   for (PetscInt refine = 0; refine < refinements; refine++) {
3426     PetscInt     m;
3427     DM           dmf;
3428     Vec          X;
3429     PetscScalar *x;
3430     PetscCall(DMRefine(dm, MPI_COMM_NULL, &dmf));
3431     PetscCall(DMPlexReplace_Internal(dm, &dmf));
3432 
3433     PetscCall(DMGetCoordinatesLocal(dm, &X));
3434     PetscCall(VecGetLocalSize(X, &m));
3435     PetscCall(VecGetArray(X, &x));
3436     for (PetscInt i = 0; i < m; i += 3) PetscCall(TPSNearestPoint(evalFunc, &x[i]));
3437     PetscCall(VecRestoreArray(X, &x));
3438   }
3439 
3440   // Face Sets has already been propagated to new vertices during refinement; this propagates to the initial vertices.
3441   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3442   PetscCall(DMPlexLabelComplete(dm, label));
3443 
3444   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
3445 
3446   if (thickness > 0) {
3447     DM              edm, cdm, ecdm;
3448     DMPlexTransform tr;
3449     const char     *prefix;
3450     PetscOptions    options;
3451     // Code from DMPlexExtrude
3452     PetscCall(DMPlexTransformCreate(PetscObjectComm((PetscObject)dm), &tr));
3453     PetscCall(DMPlexTransformSetDM(tr, dm));
3454     PetscCall(DMPlexTransformSetType(tr, DMPLEXEXTRUDE));
3455     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
3456     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)tr, prefix));
3457     PetscCall(PetscObjectGetOptions((PetscObject)dm, &options));
3458     PetscCall(PetscObjectSetOptions((PetscObject)tr, options));
3459     PetscCall(DMPlexTransformExtrudeSetLayers(tr, layers));
3460     PetscCall(DMPlexTransformExtrudeSetThickness(tr, thickness));
3461     PetscCall(DMPlexTransformExtrudeSetTensor(tr, PETSC_FALSE));
3462     PetscCall(DMPlexTransformExtrudeSetSymmetric(tr, PETSC_TRUE));
3463     PetscCall(DMPlexTransformExtrudeSetNormalFunction(tr, normalFunc));
3464     PetscCall(DMPlexTransformSetFromOptions(tr));
3465     PetscCall(PetscObjectSetOptions((PetscObject)tr, NULL));
3466     PetscCall(DMPlexTransformSetUp(tr));
3467     PetscCall(PetscObjectViewFromOptions((PetscObject)tr, NULL, "-dm_plex_tps_transform_view"));
3468     PetscCall(DMPlexTransformApply(tr, dm, &edm));
3469     PetscCall(DMCopyDisc(dm, edm));
3470     PetscCall(DMGetCoordinateDM(dm, &cdm));
3471     PetscCall(DMGetCoordinateDM(edm, &ecdm));
3472     PetscCall(DMCopyDisc(cdm, ecdm));
3473     PetscCall(DMPlexTransformCreateDiscLabels(tr, edm));
3474     PetscCall(DMPlexTransformDestroy(&tr));
3475     if (edm) {
3476       ((DM_Plex *)edm->data)->printFEM    = ((DM_Plex *)dm->data)->printFEM;
3477       ((DM_Plex *)edm->data)->printL2     = ((DM_Plex *)dm->data)->printL2;
3478       ((DM_Plex *)edm->data)->printLocate = ((DM_Plex *)dm->data)->printLocate;
3479     }
3480     PetscCall(DMPlexReplace_Internal(dm, &edm));
3481   }
3482   PetscFunctionReturn(PETSC_SUCCESS);
3483 }
3484 
3485 /*@
3486   DMPlexCreateTPSMesh - Create a distributed, interpolated mesh of a triply-periodic surface
3487 
3488   Collective
3489 
3490   Input Parameters:
3491 + comm   - The communicator for the `DM` object
3492 . tpstype - Type of triply-periodic surface
3493 . extent - Array of length 3 containing number of periods in each direction
3494 . periodic - array of length 3 with periodicity, or NULL for non-periodic
3495 . tps_distribute - Distribute 2D manifold mesh prior to refinement and extrusion (more scalable)
3496 . refinements - Number of factor-of-2 refinements of 2D manifold mesh
3497 . layers - Number of cell layers extruded in normal direction
3498 - thickness - Thickness in normal direction
3499 
3500   Output Parameter:
3501 . dm  - The `DM` object
3502 
3503   Level: beginner
3504 
3505   Notes:
3506   This meshes the surface of the Schwarz P or Gyroid surfaces.  Schwarz P is is the simplest member of the triply-periodic minimal surfaces.
3507   https://en.wikipedia.org/wiki/Schwarz_minimal_surface#Schwarz_P_(%22Primitive%22) and can be cut with "clean" boundaries.
3508   The Gyroid (https://en.wikipedia.org/wiki/Gyroid) is another triply-periodic minimal surface with applications in additive manufacturing; it is much more difficult to "cut" since there are no planes of symmetry.
3509   Our implementation creates a very coarse mesh of the surface and refines (by 4-way splitting) as many times as requested.
3510   On each refinement, all vertices are projected to their nearest point on the surface.
3511   This projection could readily be extended to related surfaces.
3512 
3513   The face (edge) sets for the Schwarz P surface are numbered 1(-x), 2(+x), 3(-y), 4(+y), 5(-z), 6(+z).
3514   When the mesh is refined, "Face Sets" contain the new vertices (created during refinement).  Use DMPlexLabelComplete() to propagate to coarse-level vertices.
3515 
3516   Developer Note:
3517   The Gyroid mesh does not currently mark boundary sets.
3518 
3519   References:
3520 . * - Maskery et al, Insights into the mechanical properties of several triply periodic minimal surface lattice structures made by polymer additive manufacturing, 2017.
3521     https://doi.org/10.1016/j.polymer.2017.11.049
3522 
3523 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMSetType()`, `DMCreate()`
3524 @*/
3525 PetscErrorCode DMPlexCreateTPSMesh(MPI_Comm comm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness, DM *dm)
3526 {
3527   PetscFunctionBegin;
3528   PetscCall(DMCreate(comm, dm));
3529   PetscCall(DMSetType(*dm, DMPLEX));
3530   PetscCall(DMPlexCreateTPSMesh_Internal(*dm, tpstype, extent, periodic, tps_distribute, refinements, layers, thickness));
3531   PetscFunctionReturn(PETSC_SUCCESS);
3532 }
3533 
3534 /*@
3535   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
3536 
3537   Collective
3538 
3539   Input Parameters:
3540 + comm    - The communicator for the `DM` object
3541 . dim     - The dimension
3542 . simplex - Use simplices, or tensor product cells
3543 - R       - The radius
3544 
3545   Output Parameter:
3546 . dm  - The `DM` object
3547 
3548   Level: beginner
3549 
3550 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBallMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3551 @*/
3552 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscReal R, DM *dm)
3553 {
3554   PetscFunctionBegin;
3555   PetscValidPointer(dm, 5);
3556   PetscCall(DMCreate(comm, dm));
3557   PetscCall(DMSetType(*dm, DMPLEX));
3558   PetscCall(DMPlexCreateSphereMesh_Internal(*dm, dim, simplex, R));
3559   PetscFunctionReturn(PETSC_SUCCESS);
3560 }
3561 
3562 static PetscErrorCode DMPlexCreateBallMesh_Internal(DM dm, PetscInt dim, PetscReal R)
3563 {
3564   DM      sdm, vol;
3565   DMLabel bdlabel;
3566 
3567   PetscFunctionBegin;
3568   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
3569   PetscCall(DMSetType(sdm, DMPLEX));
3570   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sdm, "bd_"));
3571   PetscCall(DMPlexCreateSphereMesh_Internal(sdm, dim - 1, PETSC_TRUE, R));
3572   PetscCall(DMSetFromOptions(sdm));
3573   PetscCall(DMViewFromOptions(sdm, NULL, "-dm_view"));
3574   PetscCall(DMPlexGenerate(sdm, NULL, PETSC_TRUE, &vol));
3575   PetscCall(DMDestroy(&sdm));
3576   PetscCall(DMPlexReplace_Internal(dm, &vol));
3577   PetscCall(DMCreateLabel(dm, "marker"));
3578   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
3579   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
3580   PetscCall(DMPlexLabelComplete(dm, bdlabel));
3581   PetscFunctionReturn(PETSC_SUCCESS);
3582 }
3583 
3584 /*@
3585   DMPlexCreateBallMesh - Creates a simplex mesh on the d-dimensional ball, B^d.
3586 
3587   Collective
3588 
3589   Input Parameters:
3590 + comm  - The communicator for the `DM` object
3591 . dim   - The dimension
3592 - R     - The radius
3593 
3594   Output Parameter:
3595 . dm  - The `DM` object
3596 
3597   Options Database Key:
3598 - bd_dm_refine - This will refine the surface mesh preserving the sphere geometry
3599 
3600   Level: beginner
3601 
3602 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3603 @*/
3604 PetscErrorCode DMPlexCreateBallMesh(MPI_Comm comm, PetscInt dim, PetscReal R, DM *dm)
3605 {
3606   PetscFunctionBegin;
3607   PetscCall(DMCreate(comm, dm));
3608   PetscCall(DMSetType(*dm, DMPLEX));
3609   PetscCall(DMPlexCreateBallMesh_Internal(*dm, dim, R));
3610   PetscFunctionReturn(PETSC_SUCCESS);
3611 }
3612 
3613 static PetscErrorCode DMPlexCreateReferenceCell_Internal(DM rdm, DMPolytopeType ct)
3614 {
3615   PetscFunctionBegin;
3616   switch (ct) {
3617   case DM_POLYTOPE_POINT: {
3618     PetscInt    numPoints[1]        = {1};
3619     PetscInt    coneSize[1]         = {0};
3620     PetscInt    cones[1]            = {0};
3621     PetscInt    coneOrientations[1] = {0};
3622     PetscScalar vertexCoords[1]     = {0.0};
3623 
3624     PetscCall(DMSetDimension(rdm, 0));
3625     PetscCall(DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3626   } break;
3627   case DM_POLYTOPE_SEGMENT: {
3628     PetscInt    numPoints[2]        = {2, 1};
3629     PetscInt    coneSize[3]         = {2, 0, 0};
3630     PetscInt    cones[2]            = {1, 2};
3631     PetscInt    coneOrientations[2] = {0, 0};
3632     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3633 
3634     PetscCall(DMSetDimension(rdm, 1));
3635     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3636   } break;
3637   case DM_POLYTOPE_POINT_PRISM_TENSOR: {
3638     PetscInt    numPoints[2]        = {2, 1};
3639     PetscInt    coneSize[3]         = {2, 0, 0};
3640     PetscInt    cones[2]            = {1, 2};
3641     PetscInt    coneOrientations[2] = {0, 0};
3642     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3643 
3644     PetscCall(DMSetDimension(rdm, 1));
3645     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3646   } break;
3647   case DM_POLYTOPE_TRIANGLE: {
3648     PetscInt    numPoints[2]        = {3, 1};
3649     PetscInt    coneSize[4]         = {3, 0, 0, 0};
3650     PetscInt    cones[3]            = {1, 2, 3};
3651     PetscInt    coneOrientations[3] = {0, 0, 0};
3652     PetscScalar vertexCoords[6]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0};
3653 
3654     PetscCall(DMSetDimension(rdm, 2));
3655     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3656   } break;
3657   case DM_POLYTOPE_QUADRILATERAL: {
3658     PetscInt    numPoints[2]        = {4, 1};
3659     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3660     PetscInt    cones[4]            = {1, 2, 3, 4};
3661     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3662     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0};
3663 
3664     PetscCall(DMSetDimension(rdm, 2));
3665     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3666   } break;
3667   case DM_POLYTOPE_SEG_PRISM_TENSOR: {
3668     PetscInt    numPoints[2]        = {4, 1};
3669     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3670     PetscInt    cones[4]            = {1, 2, 3, 4};
3671     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3672     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0};
3673 
3674     PetscCall(DMSetDimension(rdm, 2));
3675     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3676   } break;
3677   case DM_POLYTOPE_TETRAHEDRON: {
3678     PetscInt    numPoints[2]        = {4, 1};
3679     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3680     PetscInt    cones[4]            = {1, 2, 3, 4};
3681     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3682     PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0};
3683 
3684     PetscCall(DMSetDimension(rdm, 3));
3685     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3686   } break;
3687   case DM_POLYTOPE_HEXAHEDRON: {
3688     PetscInt    numPoints[2]        = {8, 1};
3689     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3690     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3691     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3692     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3693 
3694     PetscCall(DMSetDimension(rdm, 3));
3695     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3696   } break;
3697   case DM_POLYTOPE_TRI_PRISM: {
3698     PetscInt    numPoints[2]        = {6, 1};
3699     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3700     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3701     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3702     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3703 
3704     PetscCall(DMSetDimension(rdm, 3));
3705     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3706   } break;
3707   case DM_POLYTOPE_TRI_PRISM_TENSOR: {
3708     PetscInt    numPoints[2]        = {6, 1};
3709     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3710     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3711     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3712     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3713 
3714     PetscCall(DMSetDimension(rdm, 3));
3715     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3716   } break;
3717   case DM_POLYTOPE_QUAD_PRISM_TENSOR: {
3718     PetscInt    numPoints[2]        = {8, 1};
3719     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3720     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3721     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3722     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3723 
3724     PetscCall(DMSetDimension(rdm, 3));
3725     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3726   } break;
3727   case DM_POLYTOPE_PYRAMID: {
3728     PetscInt    numPoints[2]        = {5, 1};
3729     PetscInt    coneSize[6]         = {5, 0, 0, 0, 0, 0};
3730     PetscInt    cones[5]            = {1, 2, 3, 4, 5};
3731     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3732     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, 0.0, 0.0, 1.0};
3733 
3734     PetscCall(DMSetDimension(rdm, 3));
3735     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3736   } break;
3737   default:
3738     SETERRQ(PetscObjectComm((PetscObject)rdm), PETSC_ERR_ARG_WRONG, "Cannot create reference cell for cell type %s", DMPolytopeTypes[ct]);
3739   }
3740   {
3741     PetscInt Nv, v;
3742 
3743     /* Must create the celltype label here so that we do not automatically try to compute the types */
3744     PetscCall(DMCreateLabel(rdm, "celltype"));
3745     PetscCall(DMPlexSetCellType(rdm, 0, ct));
3746     PetscCall(DMPlexGetChart(rdm, NULL, &Nv));
3747     for (v = 1; v < Nv; ++v) PetscCall(DMPlexSetCellType(rdm, v, DM_POLYTOPE_POINT));
3748   }
3749   PetscCall(DMPlexInterpolateInPlace_Internal(rdm));
3750   PetscCall(PetscObjectSetName((PetscObject)rdm, DMPolytopeTypes[ct]));
3751   PetscFunctionReturn(PETSC_SUCCESS);
3752 }
3753 
3754 /*@
3755   DMPlexCreateReferenceCell - Create a `DMPLEX` with the appropriate FEM reference cell
3756 
3757   Collective
3758 
3759   Input Parameters:
3760 + comm - The communicator
3761 - ct   - The cell type of the reference cell
3762 
3763   Output Parameter:
3764 . refdm - The reference cell
3765 
3766   Level: intermediate
3767 
3768 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateReferenceCell()`, `DMPlexCreateBoxMesh()`
3769 @*/
3770 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, DMPolytopeType ct, DM *refdm)
3771 {
3772   PetscFunctionBegin;
3773   PetscCall(DMCreate(comm, refdm));
3774   PetscCall(DMSetType(*refdm, DMPLEX));
3775   PetscCall(DMPlexCreateReferenceCell_Internal(*refdm, ct));
3776   PetscFunctionReturn(PETSC_SUCCESS);
3777 }
3778 
3779 static PetscErrorCode DMPlexCreateBoundaryLabel_Private(DM dm, const char name[])
3780 {
3781   DM        plex;
3782   DMLabel   label;
3783   PetscBool hasLabel;
3784 
3785   PetscFunctionBegin;
3786   PetscCall(DMHasLabel(dm, name, &hasLabel));
3787   if (hasLabel) PetscFunctionReturn(PETSC_SUCCESS);
3788   PetscCall(DMCreateLabel(dm, name));
3789   PetscCall(DMGetLabel(dm, name, &label));
3790   PetscCall(DMConvert(dm, DMPLEX, &plex));
3791   PetscCall(DMPlexMarkBoundaryFaces(plex, 1, label));
3792   PetscCall(DMPlexLabelComplete(plex, label));
3793   PetscCall(DMDestroy(&plex));
3794   PetscFunctionReturn(PETSC_SUCCESS);
3795 }
3796 
3797 /*
3798   We use the last coordinate as the radius, the inner radius is lower[dim-1] and the outer radius is upper[dim-1]. Then we map the first coordinate around the circle.
3799 
3800     (x, y) -> (r, theta) = (x[1], (x[0] - lower[0]) * 2\pi/(upper[0] - lower[0]))
3801 */
3802 static void boxToAnnulus(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
3803 {
3804   const PetscReal low = PetscRealPart(constants[0]);
3805   const PetscReal upp = PetscRealPart(constants[1]);
3806   const PetscReal r   = PetscRealPart(u[1]);
3807   const PetscReal th  = 2. * PETSC_PI * (PetscRealPart(u[0]) - low) / (upp - low);
3808 
3809   f0[0] = r * PetscCosReal(th);
3810   f0[1] = r * PetscSinReal(th);
3811 }
3812 
3813 const char *const DMPlexShapes[] = {"box", "box_surface", "ball", "sphere", "cylinder", "schwarz_p", "gyroid", "doublet", "annulus", "hypercubic", "zbox", "unknown", "DMPlexShape", "DM_SHAPE_", NULL};
3814 
3815 static PetscErrorCode DMPlexCreateFromOptions_Internal(PetscOptionItems *PetscOptionsObject, PetscBool *useCoordSpace, DM dm)
3816 {
3817   DMPlexShape    shape   = DM_SHAPE_BOX;
3818   DMPolytopeType cell    = DM_POLYTOPE_TRIANGLE;
3819   PetscInt       dim     = 2;
3820   PetscBool      simplex = PETSC_TRUE, interpolate = PETSC_TRUE, adjCone = PETSC_FALSE, adjClosure = PETSC_TRUE, refDomain = PETSC_FALSE;
3821   PetscBool      flg, flg2, fflg, bdfflg, nameflg;
3822   MPI_Comm       comm;
3823   char           filename[PETSC_MAX_PATH_LEN]   = "<unspecified>";
3824   char           bdFilename[PETSC_MAX_PATH_LEN] = "<unspecified>";
3825   char           plexname[PETSC_MAX_PATH_LEN]   = "";
3826 
3827   PetscFunctionBegin;
3828   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3829   /* TODO Turn this into a registration interface */
3830   PetscCall(PetscOptionsString("-dm_plex_filename", "File containing a mesh", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &fflg));
3831   PetscCall(PetscOptionsString("-dm_plex_boundary_filename", "File containing a mesh boundary", "DMPlexCreateFromFile", bdFilename, bdFilename, sizeof(bdFilename), &bdfflg));
3832   PetscCall(PetscOptionsString("-dm_plex_name", "Name of the mesh in the file", "DMPlexCreateFromFile", plexname, plexname, sizeof(plexname), &nameflg));
3833   PetscCall(PetscOptionsEnum("-dm_plex_cell", "Cell shape", "", DMPolytopeTypes, (PetscEnum)cell, (PetscEnum *)&cell, NULL));
3834   PetscCall(PetscOptionsBool("-dm_plex_reference_cell_domain", "Use a reference cell domain", "", refDomain, &refDomain, NULL));
3835   PetscCall(PetscOptionsEnum("-dm_plex_shape", "Shape for built-in mesh", "", DMPlexShapes, (PetscEnum)shape, (PetscEnum *)&shape, &flg));
3836   PetscCall(PetscOptionsBoundedInt("-dm_plex_dim", "Topological dimension of the mesh", "DMGetDimension", dim, &dim, &flg, 0));
3837   PetscCheck(dim >= 0, comm, PETSC_ERR_ARG_OUTOFRANGE, "Dimension %" PetscInt_FMT " should be in [0, infinity)", dim);
3838   PetscCall(PetscOptionsBool("-dm_plex_simplex", "Mesh cell shape", "", simplex, &simplex, &flg));
3839   PetscCall(PetscOptionsBool("-dm_plex_interpolate", "Flag to create edges and faces automatically", "", interpolate, &interpolate, &flg));
3840   PetscCall(PetscOptionsBool("-dm_plex_adj_cone", "Set adjacency direction", "DMSetBasicAdjacency", adjCone, &adjCone, &flg));
3841   PetscCall(PetscOptionsBool("-dm_plex_adj_closure", "Set adjacency size", "DMSetBasicAdjacency", adjClosure, &adjClosure, &flg2));
3842   if (flg || flg2) PetscCall(DMSetBasicAdjacency(dm, adjCone, adjClosure));
3843 
3844   switch (cell) {
3845   case DM_POLYTOPE_POINT:
3846   case DM_POLYTOPE_SEGMENT:
3847   case DM_POLYTOPE_POINT_PRISM_TENSOR:
3848   case DM_POLYTOPE_TRIANGLE:
3849   case DM_POLYTOPE_QUADRILATERAL:
3850   case DM_POLYTOPE_TETRAHEDRON:
3851   case DM_POLYTOPE_HEXAHEDRON:
3852     *useCoordSpace = PETSC_TRUE;
3853     break;
3854   default:
3855     *useCoordSpace = PETSC_FALSE;
3856     break;
3857   }
3858 
3859   if (fflg) {
3860     DM dmnew;
3861 
3862     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), filename, plexname, interpolate, &dmnew));
3863     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3864     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3865   } else if (refDomain) {
3866     PetscCall(DMPlexCreateReferenceCell_Internal(dm, cell));
3867   } else if (bdfflg) {
3868     DM bdm, dmnew;
3869 
3870     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), bdFilename, plexname, interpolate, &bdm));
3871     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)bdm, "bd_"));
3872     PetscCall(DMSetFromOptions(bdm));
3873     PetscCall(DMPlexGenerate(bdm, NULL, interpolate, &dmnew));
3874     PetscCall(DMDestroy(&bdm));
3875     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3876     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3877   } else {
3878     PetscCall(PetscObjectSetName((PetscObject)dm, DMPlexShapes[shape]));
3879     switch (shape) {
3880     case DM_SHAPE_BOX:
3881     case DM_SHAPE_ZBOX:
3882     case DM_SHAPE_ANNULUS: {
3883       PetscInt       faces[3]  = {0, 0, 0};
3884       PetscReal      lower[3]  = {0, 0, 0};
3885       PetscReal      upper[3]  = {1, 1, 1};
3886       DMBoundaryType bdt[3]    = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3887       PetscBool      isAnnular = shape == DM_SHAPE_ANNULUS ? PETSC_TRUE : PETSC_FALSE;
3888       PetscInt       i, n;
3889 
3890       n = dim;
3891       for (i = 0; i < dim; ++i) faces[i] = (dim == 1 ? 1 : 4 - dim);
3892       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3893       n = 3;
3894       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3895       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3896       n = 3;
3897       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3898       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3899       n = 3;
3900       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
3901       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3902 
3903       PetscCheck(!isAnnular || dim == 2, comm, PETSC_ERR_ARG_OUTOFRANGE, "Only two dimensional annuli have been implemented");
3904       if (isAnnular)
3905         for (i = 0; i < dim - 1; ++i) bdt[i] = DM_BOUNDARY_PERIODIC;
3906 
3907       switch (cell) {
3908       case DM_POLYTOPE_TRI_PRISM_TENSOR:
3909         PetscCall(DMPlexCreateWedgeBoxMesh_Internal(dm, faces, lower, upper, bdt));
3910         if (!interpolate) {
3911           DM udm;
3912 
3913           PetscCall(DMPlexUninterpolate(dm, &udm));
3914           PetscCall(DMPlexReplace_Internal(dm, &udm));
3915         }
3916         break;
3917       default:
3918         PetscCall(DMPlexCreateBoxMesh_Internal(dm, shape, dim, simplex, faces, lower, upper, bdt, interpolate));
3919         break;
3920       }
3921       if (isAnnular) {
3922         DM          cdm;
3923         PetscDS     cds;
3924         PetscScalar bounds[2] = {lower[0], upper[0]};
3925 
3926         // Fix coordinates for annular region
3927         PetscCall(DMSetPeriodicity(dm, NULL, NULL, NULL));
3928         PetscCall(DMSetCellCoordinatesLocal(dm, NULL));
3929         PetscCall(DMSetCellCoordinates(dm, NULL));
3930         PetscCall(DMPlexCreateCoordinateSpace(dm, 1, NULL));
3931         PetscCall(DMGetCoordinateDM(dm, &cdm));
3932         PetscCall(DMGetDS(cdm, &cds));
3933         PetscCall(PetscDSSetConstants(cds, 2, bounds));
3934         PetscCall(DMPlexRemapGeometry(dm, 0.0, boxToAnnulus));
3935       }
3936     } break;
3937     case DM_SHAPE_BOX_SURFACE: {
3938       PetscInt  faces[3] = {0, 0, 0};
3939       PetscReal lower[3] = {0, 0, 0};
3940       PetscReal upper[3] = {1, 1, 1};
3941       PetscInt  i, n;
3942 
3943       n = dim + 1;
3944       for (i = 0; i < dim + 1; ++i) faces[i] = (dim + 1 == 1 ? 1 : 4 - (dim + 1));
3945       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3946       n = 3;
3947       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3948       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3949       n = 3;
3950       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3951       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3952       PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(dm, dim + 1, faces, lower, upper, interpolate));
3953     } break;
3954     case DM_SHAPE_SPHERE: {
3955       PetscReal R = 1.0;
3956 
3957       PetscCall(PetscOptionsReal("-dm_plex_sphere_radius", "Radius of the sphere", "", R, &R, &flg));
3958       PetscCall(DMPlexCreateSphereMesh_Internal(dm, dim, simplex, R));
3959     } break;
3960     case DM_SHAPE_BALL: {
3961       PetscReal R = 1.0;
3962 
3963       PetscCall(PetscOptionsReal("-dm_plex_ball_radius", "Radius of the ball", "", R, &R, &flg));
3964       PetscCall(DMPlexCreateBallMesh_Internal(dm, dim, R));
3965     } break;
3966     case DM_SHAPE_CYLINDER: {
3967       DMBoundaryType bdt = DM_BOUNDARY_NONE;
3968       PetscInt       Nw  = 6;
3969 
3970       PetscCall(PetscOptionsEnum("-dm_plex_cylinder_bd", "Boundary type in the z direction", "", DMBoundaryTypes, (PetscEnum)bdt, (PetscEnum *)&bdt, NULL));
3971       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_wedges", "Number of wedges around the cylinder", "", Nw, &Nw, NULL));
3972       switch (cell) {
3973       case DM_POLYTOPE_TRI_PRISM_TENSOR:
3974         PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(dm, Nw, interpolate));
3975         break;
3976       default:
3977         PetscCall(DMPlexCreateHexCylinderMesh_Internal(dm, bdt));
3978         break;
3979       }
3980     } break;
3981     case DM_SHAPE_SCHWARZ_P: // fallthrough
3982     case DM_SHAPE_GYROID: {
3983       PetscInt       extent[3] = {1, 1, 1}, refine = 0, layers = 0, three;
3984       PetscReal      thickness   = 0.;
3985       DMBoundaryType periodic[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3986       DMPlexTPSType  tps_type    = shape == DM_SHAPE_SCHWARZ_P ? DMPLEX_TPS_SCHWARZ_P : DMPLEX_TPS_GYROID;
3987       PetscBool      tps_distribute;
3988       PetscCall(PetscOptionsIntArray("-dm_plex_tps_extent", "Number of replicas for each of three dimensions", NULL, extent, (three = 3, &three), NULL));
3989       PetscCall(PetscOptionsInt("-dm_plex_tps_refine", "Number of refinements", NULL, refine, &refine, NULL));
3990       PetscCall(PetscOptionsEnumArray("-dm_plex_tps_periodic", "Periodicity in each of three dimensions", NULL, DMBoundaryTypes, (PetscEnum *)periodic, (three = 3, &three), NULL));
3991       PetscCall(PetscOptionsInt("-dm_plex_tps_layers", "Number of layers in volumetric extrusion (or zero to not extrude)", NULL, layers, &layers, NULL));
3992       PetscCall(PetscOptionsReal("-dm_plex_tps_thickness", "Thickness of volumetric extrusion", NULL, thickness, &thickness, NULL));
3993       PetscCall(DMPlexDistributeGetDefault(dm, &tps_distribute));
3994       PetscCall(PetscOptionsBool("-dm_plex_tps_distribute", "Distribute the 2D mesh prior to refinement and extrusion", NULL, tps_distribute, &tps_distribute, NULL));
3995       PetscCall(DMPlexCreateTPSMesh_Internal(dm, tps_type, extent, periodic, tps_distribute, refine, layers, thickness));
3996     } break;
3997     case DM_SHAPE_DOUBLET: {
3998       DM        dmnew;
3999       PetscReal rl = 0.0;
4000 
4001       PetscCall(PetscOptionsReal("-dm_plex_doublet_refinementlimit", "Refinement limit", NULL, rl, &rl, NULL));
4002       PetscCall(DMPlexCreateDoublet(PetscObjectComm((PetscObject)dm), dim, simplex, interpolate, rl, &dmnew));
4003       PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4004       PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4005     } break;
4006     case DM_SHAPE_HYPERCUBIC: {
4007       PetscInt       *edges;
4008       PetscReal      *lower, *upper;
4009       DMBoundaryType *bdt;
4010       PetscInt        n, d;
4011 
4012       *useCoordSpace = PETSC_FALSE;
4013       PetscCall(PetscMalloc4(dim, &edges, dim, &lower, dim, &upper, dim, &bdt));
4014       for (d = 0; d < dim; ++d) {
4015         edges[d] = 1;
4016         lower[d] = 0.;
4017         upper[d] = 1.;
4018         bdt[d]   = DM_BOUNDARY_PERIODIC;
4019       }
4020       n = dim;
4021       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", edges, &n, &flg));
4022       n = dim;
4023       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
4024       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4025       n = dim;
4026       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
4027       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4028       n = dim;
4029       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
4030       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4031       PetscCall(DMPlexCreateHypercubicMesh_Internal(dm, dim, lower, upper, edges, bdt));
4032       PetscCall(PetscFree4(edges, lower, upper, bdt));
4033     } break;
4034     default:
4035       SETERRQ(comm, PETSC_ERR_SUP, "Domain shape %s is unsupported", DMPlexShapes[shape]);
4036     }
4037   }
4038   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
4039   if (!((PetscObject)dm)->name && nameflg) PetscCall(PetscObjectSetName((PetscObject)dm, plexname));
4040   PetscFunctionReturn(PETSC_SUCCESS);
4041 }
4042 
4043 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
4044 {
4045   DM_Plex  *mesh = (DM_Plex *)dm->data;
4046   PetscBool flg, flg2;
4047   char      bdLabel[PETSC_MAX_PATH_LEN];
4048 
4049   PetscFunctionBegin;
4050   /* Handle viewing */
4051   PetscCall(PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL));
4052   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fem", "Debug output level all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL, 0));
4053   PetscCall(PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL));
4054   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL, 0));
4055   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_locate", "Debug output level all point location computations", "DMLocatePoints", 0, &mesh->printLocate, NULL, 0));
4056   PetscCall(DMMonitorSetFromOptions(dm, "-dm_plex_monitor_throughput", "Monitor the simulation throughput", "DMPlexMonitorThroughput", DMPlexMonitorThroughput, NULL, &flg));
4057   if (flg) PetscCall(PetscLogDefaultBegin());
4058   /* Labeling */
4059   PetscCall(PetscOptionsString("-dm_plex_boundary_label", "Label to mark the mesh boundary", "", bdLabel, bdLabel, sizeof(bdLabel), &flg));
4060   if (flg) PetscCall(DMPlexCreateBoundaryLabel_Private(dm, bdLabel));
4061   /* Point Location */
4062   PetscCall(PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL));
4063   /* Partitioning and distribution */
4064   PetscCall(PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL));
4065   /* Generation and remeshing */
4066   PetscCall(PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL));
4067   /* Projection behavior */
4068   PetscCall(PetscOptionsBoundedInt("-dm_plex_max_projection_height", "Maximum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL, 0));
4069   PetscCall(PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL));
4070   /* Checking structure */
4071   {
4072     PetscBool all = PETSC_FALSE;
4073 
4074     PetscCall(PetscOptionsBool("-dm_plex_check_all", "Perform all basic checks", "DMPlexCheck", PETSC_FALSE, &all, NULL));
4075     if (all) {
4076       PetscCall(DMPlexCheck(dm));
4077     } else {
4078       PetscCall(PetscOptionsBool("-dm_plex_check_symmetry", "Check that the adjacency information in the mesh is symmetric", "DMPlexCheckSymmetry", PETSC_FALSE, &flg, &flg2));
4079       if (flg && flg2) PetscCall(DMPlexCheckSymmetry(dm));
4080       PetscCall(PetscOptionsBool("-dm_plex_check_skeleton", "Check that each cell has the correct number of vertices (only for homogeneous simplex or tensor meshes)", "DMPlexCheckSkeleton", PETSC_FALSE, &flg, &flg2));
4081       if (flg && flg2) PetscCall(DMPlexCheckSkeleton(dm, 0));
4082       PetscCall(PetscOptionsBool("-dm_plex_check_faces", "Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type", "DMPlexCheckFaces", PETSC_FALSE, &flg, &flg2));
4083       if (flg && flg2) PetscCall(DMPlexCheckFaces(dm, 0));
4084       PetscCall(PetscOptionsBool("-dm_plex_check_geometry", "Check that cells have positive volume", "DMPlexCheckGeometry", PETSC_FALSE, &flg, &flg2));
4085       if (flg && flg2) PetscCall(DMPlexCheckGeometry(dm));
4086       PetscCall(PetscOptionsBool("-dm_plex_check_pointsf", "Check some necessary conditions for PointSF", "DMPlexCheckPointSF", PETSC_FALSE, &flg, &flg2));
4087       if (flg && flg2) PetscCall(DMPlexCheckPointSF(dm, NULL, PETSC_FALSE));
4088       PetscCall(PetscOptionsBool("-dm_plex_check_interface_cones", "Check points on inter-partition interfaces have conforming order of cone points", "DMPlexCheckInterfaceCones", PETSC_FALSE, &flg, &flg2));
4089       if (flg && flg2) PetscCall(DMPlexCheckInterfaceCones(dm));
4090     }
4091     PetscCall(PetscOptionsBool("-dm_plex_check_cell_shape", "Check cell shape", "DMPlexCheckCellShape", PETSC_FALSE, &flg, &flg2));
4092     if (flg && flg2) PetscCall(DMPlexCheckCellShape(dm, PETSC_TRUE, PETSC_DETERMINE));
4093   }
4094   {
4095     PetscReal scale = 1.0;
4096 
4097     PetscCall(PetscOptionsReal("-dm_plex_scale", "Scale factor for mesh coordinates", "DMPlexScale", scale, &scale, &flg));
4098     if (flg) {
4099       Vec coordinates, coordinatesLocal;
4100 
4101       PetscCall(DMGetCoordinates(dm, &coordinates));
4102       PetscCall(DMGetCoordinatesLocal(dm, &coordinatesLocal));
4103       PetscCall(VecScale(coordinates, scale));
4104       PetscCall(VecScale(coordinatesLocal, scale));
4105     }
4106   }
4107   PetscCall(PetscPartitionerSetFromOptions(mesh->partitioner));
4108   PetscFunctionReturn(PETSC_SUCCESS);
4109 }
4110 
4111 PetscErrorCode DMSetFromOptions_Overlap_Plex(DM dm, PetscOptionItems *PetscOptionsObject, PetscInt *overlap)
4112 {
4113   PetscInt  numOvLabels = 16, numOvExLabels = 16;
4114   char     *ovLabelNames[16], *ovExLabelNames[16];
4115   PetscInt  numOvValues = 16, numOvExValues = 16, l;
4116   PetscBool flg;
4117 
4118   PetscFunctionBegin;
4119   PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", *overlap, overlap, NULL, 0));
4120   PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_labels", "List of overlap label names", "DMPlexDistribute", ovLabelNames, &numOvLabels, &flg));
4121   if (!flg) numOvLabels = 0;
4122   if (numOvLabels) {
4123     ((DM_Plex *)dm->data)->numOvLabels = numOvLabels;
4124     for (l = 0; l < numOvLabels; ++l) {
4125       PetscCall(DMGetLabel(dm, ovLabelNames[l], &((DM_Plex *)dm->data)->ovLabels[l]));
4126       PetscCheck(((DM_Plex *)dm->data)->ovLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovLabelNames[l]);
4127       PetscCall(PetscFree(ovLabelNames[l]));
4128     }
4129     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_values", "List of overlap label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovValues, &numOvValues, &flg));
4130     if (!flg) numOvValues = 0;
4131     PetscCheck(numOvLabels == numOvValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvLabels, numOvValues);
4132 
4133     PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_exclude_labels", "List of overlap exclude label names", "DMPlexDistribute", ovExLabelNames, &numOvExLabels, &flg));
4134     if (!flg) numOvExLabels = 0;
4135     ((DM_Plex *)dm->data)->numOvExLabels = numOvExLabels;
4136     for (l = 0; l < numOvExLabels; ++l) {
4137       PetscCall(DMGetLabel(dm, ovExLabelNames[l], &((DM_Plex *)dm->data)->ovExLabels[l]));
4138       PetscCheck(((DM_Plex *)dm->data)->ovExLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovExLabelNames[l]);
4139       PetscCall(PetscFree(ovExLabelNames[l]));
4140     }
4141     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_exclude_values", "List of overlap exclude label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovExValues, &numOvExValues, &flg));
4142     if (!flg) numOvExValues = 0;
4143     PetscCheck(numOvExLabels == numOvExValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of exclude labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvExLabels, numOvExValues);
4144   }
4145   PetscFunctionReturn(PETSC_SUCCESS);
4146 }
4147 
4148 static PetscErrorCode DMSetFromOptions_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
4149 {
4150   PetscFunctionList        ordlist;
4151   char                     oname[256];
4152   PetscReal                volume    = -1.0;
4153   PetscInt                 prerefine = 0, refine = 0, r, coarsen = 0, overlap = 0, extLayers = 0, dim;
4154   PetscBool                uniformOrig, created = PETSC_FALSE, uniform = PETSC_TRUE, distribute, interpolate = PETSC_TRUE, coordSpace = PETSC_TRUE, remap = PETSC_TRUE, ghostCells = PETSC_FALSE, isHierarchy, ignoreModel = PETSC_FALSE, flg;
4155   DMPlexReorderDefaultFlag reorder;
4156 
4157   PetscFunctionBegin;
4158   PetscOptionsHeadBegin(PetscOptionsObject, "DMPlex Options");
4159   if (dm->cloneOpts) goto non_refine;
4160   /* Handle automatic creation */
4161   PetscCall(DMGetDimension(dm, &dim));
4162   if (dim < 0) {
4163     PetscCall(DMPlexCreateFromOptions_Internal(PetscOptionsObject, &coordSpace, dm));
4164     created = PETSC_TRUE;
4165   }
4166   PetscCall(DMGetDimension(dm, &dim));
4167   /* Handle interpolation before distribution */
4168   PetscCall(PetscOptionsBool("-dm_plex_interpolate_pre", "Flag to interpolate mesh before distribution", "", interpolate, &interpolate, &flg));
4169   if (flg) {
4170     DMPlexInterpolatedFlag interpolated;
4171 
4172     PetscCall(DMPlexIsInterpolated(dm, &interpolated));
4173     if (interpolated == DMPLEX_INTERPOLATED_FULL && !interpolate) {
4174       DM udm;
4175 
4176       PetscCall(DMPlexUninterpolate(dm, &udm));
4177       PetscCall(DMPlexReplace_Internal(dm, &udm));
4178     } else if (interpolated != DMPLEX_INTERPOLATED_FULL && interpolate) {
4179       DM idm;
4180 
4181       PetscCall(DMPlexInterpolate(dm, &idm));
4182       PetscCall(DMPlexReplace_Internal(dm, &idm));
4183     }
4184   }
4185   /* Handle DMPlex refinement before distribution */
4186   PetscCall(PetscOptionsBool("-dm_refine_ignore_model", "Flag to ignore the geometry model when refining", "DMCreate", ignoreModel, &ignoreModel, &flg));
4187   if (flg) ((DM_Plex *)dm->data)->ignoreModel = ignoreModel;
4188   PetscCall(DMPlexGetRefinementUniform(dm, &uniformOrig));
4189   PetscCall(PetscOptionsBoundedInt("-dm_refine_pre", "The number of refinements before distribution", "DMCreate", prerefine, &prerefine, NULL, 0));
4190   PetscCall(PetscOptionsBool("-dm_refine_remap_pre", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
4191   PetscCall(PetscOptionsBool("-dm_refine_uniform_pre", "Flag for uniform refinement before distribution", "DMCreate", uniform, &uniform, &flg));
4192   if (flg) PetscCall(DMPlexSetRefinementUniform(dm, uniform));
4193   PetscCall(PetscOptionsReal("-dm_refine_volume_limit_pre", "The maximum cell volume after refinement before distribution", "DMCreate", volume, &volume, &flg));
4194   if (flg) {
4195     PetscCall(DMPlexSetRefinementUniform(dm, PETSC_FALSE));
4196     PetscCall(DMPlexSetRefinementLimit(dm, volume));
4197     prerefine = PetscMax(prerefine, 1);
4198   }
4199   for (r = 0; r < prerefine; ++r) {
4200     DM             rdm;
4201     PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4202 
4203     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4204     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
4205     PetscCall(DMPlexReplace_Internal(dm, &rdm));
4206     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4207     if (coordFunc && remap) {
4208       PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4209       ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4210     }
4211   }
4212   PetscCall(DMPlexSetRefinementUniform(dm, uniformOrig));
4213   /* Handle DMPlex extrusion before distribution */
4214   PetscCall(PetscOptionsBoundedInt("-dm_extrude", "The number of layers to extrude", "", extLayers, &extLayers, NULL, 0));
4215   if (extLayers) {
4216     DM edm;
4217 
4218     PetscCall(DMExtrude(dm, extLayers, &edm));
4219     PetscCall(DMPlexReplace_Internal(dm, &edm));
4220     ((DM_Plex *)dm->data)->coordFunc = NULL;
4221     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4222     extLayers = 0;
4223     PetscCall(DMGetDimension(dm, &dim));
4224   }
4225   /* Handle DMPlex reordering before distribution */
4226   PetscCall(DMPlexReorderGetDefault(dm, &reorder));
4227   PetscCall(MatGetOrderingList(&ordlist));
4228   PetscCall(PetscStrncpy(oname, MATORDERINGNATURAL, sizeof(oname)));
4229   PetscCall(PetscOptionsFList("-dm_plex_reorder", "Set mesh reordering type", "DMPlexGetOrdering", ordlist, MATORDERINGNATURAL, oname, sizeof(oname), &flg));
4230   if (reorder == DMPLEX_REORDER_DEFAULT_TRUE || flg) {
4231     DM pdm;
4232     IS perm;
4233 
4234     PetscCall(DMPlexGetOrdering(dm, oname, NULL, &perm));
4235     PetscCall(DMPlexPermute(dm, perm, &pdm));
4236     PetscCall(ISDestroy(&perm));
4237     PetscCall(DMPlexReplace_Internal(dm, &pdm));
4238     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4239   }
4240   /* Handle DMPlex distribution */
4241   PetscCall(DMPlexDistributeGetDefault(dm, &distribute));
4242   PetscCall(PetscOptionsBool("-dm_distribute", "Flag to redistribute a mesh among processes", "DMPlexDistribute", distribute, &distribute, NULL));
4243   PetscCall(DMSetFromOptions_Overlap_Plex(dm, PetscOptionsObject, &overlap));
4244   if (distribute) {
4245     DM               pdm = NULL;
4246     PetscPartitioner part;
4247 
4248     PetscCall(DMPlexGetPartitioner(dm, &part));
4249     PetscCall(PetscPartitionerSetFromOptions(part));
4250     PetscCall(DMPlexDistribute(dm, overlap, NULL, &pdm));
4251     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
4252   }
4253   /* Create coordinate space */
4254   if (created) {
4255     DM_Plex  *mesh   = (DM_Plex *)dm->data;
4256     PetscInt  degree = 1;
4257     PetscInt  height = 0;
4258     DM        cdm;
4259     PetscBool flg;
4260 
4261     PetscCall(PetscOptionsBool("-dm_coord_space", "Use an FEM space for coordinates", "", coordSpace, &coordSpace, &flg));
4262     PetscCall(PetscOptionsInt("-dm_coord_petscspace_degree", "FEM degree for coordinate space", "", degree, &degree, NULL));
4263     if (coordSpace) PetscCall(DMPlexCreateCoordinateSpace(dm, degree, mesh->coordFunc));
4264     PetscCall(DMGetCoordinateDM(dm, &cdm));
4265     if (flg && !coordSpace) {
4266       PetscDS      cds;
4267       PetscObject  obj;
4268       PetscClassId id;
4269 
4270       PetscCall(DMGetDS(cdm, &cds));
4271       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
4272       PetscCall(PetscObjectGetClassId(obj, &id));
4273       if (id == PETSCFE_CLASSID) {
4274         PetscContainer dummy;
4275 
4276         PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &dummy));
4277         PetscCall(PetscObjectSetName((PetscObject)dummy, "coordinates"));
4278         PetscCall(DMSetField(cdm, 0, NULL, (PetscObject)dummy));
4279         PetscCall(PetscContainerDestroy(&dummy));
4280         PetscCall(DMClearDS(cdm));
4281       }
4282       mesh->coordFunc = NULL;
4283     }
4284     PetscCall(PetscOptionsBool("-dm_sparse_localize", "Localize only necessary cells", "", dm->sparseLocalize, &dm->sparseLocalize, &flg));
4285     PetscCall(PetscOptionsInt("-dm_localize_height", "Localize edges and faces in addition to cells", "", height, &height, &flg));
4286     if (flg) PetscCall(DMPlexSetMaxProjectionHeight(cdm, height));
4287     PetscCall(DMLocalizeCoordinates(dm));
4288   }
4289   /* Handle DMPlex refinement */
4290   remap = PETSC_TRUE;
4291   PetscCall(PetscOptionsBoundedInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL, 0));
4292   PetscCall(PetscOptionsBool("-dm_refine_remap", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
4293   PetscCall(PetscOptionsBoundedInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy, 0));
4294   if (refine) PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
4295   if (refine && isHierarchy) {
4296     DM *dms, coarseDM;
4297 
4298     PetscCall(DMGetCoarseDM(dm, &coarseDM));
4299     PetscCall(PetscObjectReference((PetscObject)coarseDM));
4300     PetscCall(PetscMalloc1(refine, &dms));
4301     PetscCall(DMRefineHierarchy(dm, refine, dms));
4302     /* Total hack since we do not pass in a pointer */
4303     PetscCall(DMPlexSwap_Static(dm, dms[refine - 1]));
4304     if (refine == 1) {
4305       PetscCall(DMSetCoarseDM(dm, dms[0]));
4306       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4307     } else {
4308       PetscCall(DMSetCoarseDM(dm, dms[refine - 2]));
4309       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4310       PetscCall(DMSetCoarseDM(dms[0], dms[refine - 1]));
4311       PetscCall(DMPlexSetRegularRefinement(dms[0], PETSC_TRUE));
4312     }
4313     PetscCall(DMSetCoarseDM(dms[refine - 1], coarseDM));
4314     PetscCall(PetscObjectDereference((PetscObject)coarseDM));
4315     /* Free DMs */
4316     for (r = 0; r < refine; ++r) {
4317       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4318       PetscCall(DMDestroy(&dms[r]));
4319     }
4320     PetscCall(PetscFree(dms));
4321   } else {
4322     for (r = 0; r < refine; ++r) {
4323       DM             rdm;
4324       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4325 
4326       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4327       PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
4328       /* Total hack since we do not pass in a pointer */
4329       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4330       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4331       if (coordFunc && remap) {
4332         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4333         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4334       }
4335     }
4336   }
4337   /* Handle DMPlex coarsening */
4338   PetscCall(PetscOptionsBoundedInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL, 0));
4339   PetscCall(PetscOptionsBoundedInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy, 0));
4340   if (coarsen && isHierarchy) {
4341     DM *dms;
4342 
4343     PetscCall(PetscMalloc1(coarsen, &dms));
4344     PetscCall(DMCoarsenHierarchy(dm, coarsen, dms));
4345     /* Free DMs */
4346     for (r = 0; r < coarsen; ++r) {
4347       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4348       PetscCall(DMDestroy(&dms[r]));
4349     }
4350     PetscCall(PetscFree(dms));
4351   } else {
4352     for (r = 0; r < coarsen; ++r) {
4353       DM             cdm;
4354       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4355 
4356       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4357       PetscCall(DMCoarsen(dm, PetscObjectComm((PetscObject)dm), &cdm));
4358       /* Total hack since we do not pass in a pointer */
4359       PetscCall(DMPlexReplace_Internal(dm, &cdm));
4360       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4361       if (coordFunc) {
4362         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4363         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4364       }
4365     }
4366   }
4367   /* Handle ghost cells */
4368   PetscCall(PetscOptionsBool("-dm_plex_create_fv_ghost_cells", "Flag to create finite volume ghost cells on the boundary", "DMCreate", ghostCells, &ghostCells, NULL));
4369   if (ghostCells) {
4370     DM   gdm;
4371     char lname[PETSC_MAX_PATH_LEN];
4372 
4373     lname[0] = '\0';
4374     PetscCall(PetscOptionsString("-dm_plex_fv_ghost_cells_label", "Label name for ghost cells boundary", "DMCreate", lname, lname, sizeof(lname), &flg));
4375     PetscCall(DMPlexConstructGhostCells(dm, flg ? lname : NULL, NULL, &gdm));
4376     PetscCall(DMPlexReplace_Internal(dm, &gdm));
4377   }
4378   /* Handle 1D order */
4379   if (reorder != DMPLEX_REORDER_DEFAULT_FALSE && dim == 1) {
4380     DM           cdm, rdm;
4381     PetscDS      cds;
4382     PetscObject  obj;
4383     PetscClassId id = PETSC_OBJECT_CLASSID;
4384     IS           perm;
4385     PetscInt     Nf;
4386     PetscBool    distributed;
4387 
4388     PetscCall(DMPlexIsDistributed(dm, &distributed));
4389     PetscCall(DMGetCoordinateDM(dm, &cdm));
4390     PetscCall(DMGetDS(cdm, &cds));
4391     PetscCall(PetscDSGetNumFields(cds, &Nf));
4392     if (Nf) {
4393       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
4394       PetscCall(PetscObjectGetClassId(obj, &id));
4395     }
4396     if (!distributed && id != PETSCFE_CLASSID) {
4397       PetscCall(DMPlexGetOrdering1D(dm, &perm));
4398       PetscCall(DMPlexPermute(dm, perm, &rdm));
4399       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4400       PetscCall(ISDestroy(&perm));
4401     }
4402   }
4403 /* Handle */
4404 non_refine:
4405   PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4406   PetscOptionsHeadEnd();
4407   PetscFunctionReturn(PETSC_SUCCESS);
4408 }
4409 
4410 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm, Vec *vec)
4411 {
4412   PetscFunctionBegin;
4413   PetscCall(DMCreateGlobalVector_Section_Private(dm, vec));
4414   /* PetscCall(VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM)); */
4415   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex));
4416   PetscCall(VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void))VecView_Plex_Native));
4417   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex));
4418   PetscCall(VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void))VecLoad_Plex_Native));
4419   PetscFunctionReturn(PETSC_SUCCESS);
4420 }
4421 
4422 static PetscErrorCode DMCreateLocalVector_Plex(DM dm, Vec *vec)
4423 {
4424   PetscFunctionBegin;
4425   PetscCall(DMCreateLocalVector_Section_Private(dm, vec));
4426   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex_Local));
4427   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex_Local));
4428   PetscFunctionReturn(PETSC_SUCCESS);
4429 }
4430 
4431 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
4432 {
4433   PetscInt depth, d;
4434 
4435   PetscFunctionBegin;
4436   PetscCall(DMPlexGetDepth(dm, &depth));
4437   if (depth == 1) {
4438     PetscCall(DMGetDimension(dm, &d));
4439     if (dim == 0) PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4440     else if (dim == d) PetscCall(DMPlexGetDepthStratum(dm, 1, pStart, pEnd));
4441     else {
4442       *pStart = 0;
4443       *pEnd   = 0;
4444     }
4445   } else {
4446     PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4447   }
4448   PetscFunctionReturn(PETSC_SUCCESS);
4449 }
4450 
4451 static PetscErrorCode DMGetNeighbors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
4452 {
4453   PetscSF            sf;
4454   PetscInt           niranks, njranks, n;
4455   const PetscMPIInt *iranks, *jranks;
4456   DM_Plex           *data = (DM_Plex *)dm->data;
4457 
4458   PetscFunctionBegin;
4459   PetscCall(DMGetPointSF(dm, &sf));
4460   if (!data->neighbors) {
4461     PetscCall(PetscSFSetUp(sf));
4462     PetscCall(PetscSFGetRootRanks(sf, &njranks, &jranks, NULL, NULL, NULL));
4463     PetscCall(PetscSFGetLeafRanks(sf, &niranks, &iranks, NULL, NULL));
4464     PetscCall(PetscMalloc1(njranks + niranks + 1, &data->neighbors));
4465     PetscCall(PetscArraycpy(data->neighbors + 1, jranks, njranks));
4466     PetscCall(PetscArraycpy(data->neighbors + njranks + 1, iranks, niranks));
4467     n = njranks + niranks;
4468     PetscCall(PetscSortRemoveDupsMPIInt(&n, data->neighbors + 1));
4469     /* The following cast should never fail: can't have more neighbors than PETSC_MPI_INT_MAX */
4470     PetscCall(PetscMPIIntCast(n, data->neighbors));
4471   }
4472   if (nranks) *nranks = data->neighbors[0];
4473   if (ranks) {
4474     if (data->neighbors[0]) *ranks = data->neighbors + 1;
4475     else *ranks = NULL;
4476   }
4477   PetscFunctionReturn(PETSC_SUCCESS);
4478 }
4479 
4480 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM, DM, Mat, Vec, Vec);
4481 
4482 static PetscErrorCode DMInitialize_Plex(DM dm)
4483 {
4484   PetscFunctionBegin;
4485   dm->ops->view                      = DMView_Plex;
4486   dm->ops->load                      = DMLoad_Plex;
4487   dm->ops->setfromoptions            = DMSetFromOptions_Plex;
4488   dm->ops->clone                     = DMClone_Plex;
4489   dm->ops->setup                     = DMSetUp_Plex;
4490   dm->ops->createlocalsection        = DMCreateLocalSection_Plex;
4491   dm->ops->createdefaultconstraints  = DMCreateDefaultConstraints_Plex;
4492   dm->ops->createglobalvector        = DMCreateGlobalVector_Plex;
4493   dm->ops->createlocalvector         = DMCreateLocalVector_Plex;
4494   dm->ops->getlocaltoglobalmapping   = NULL;
4495   dm->ops->createfieldis             = NULL;
4496   dm->ops->createcoordinatedm        = DMCreateCoordinateDM_Plex;
4497   dm->ops->createcoordinatefield     = DMCreateCoordinateField_Plex;
4498   dm->ops->getcoloring               = NULL;
4499   dm->ops->creatematrix              = DMCreateMatrix_Plex;
4500   dm->ops->createinterpolation       = DMCreateInterpolation_Plex;
4501   dm->ops->createmassmatrix          = DMCreateMassMatrix_Plex;
4502   dm->ops->createmassmatrixlumped    = DMCreateMassMatrixLumped_Plex;
4503   dm->ops->createinjection           = DMCreateInjection_Plex;
4504   dm->ops->refine                    = DMRefine_Plex;
4505   dm->ops->coarsen                   = DMCoarsen_Plex;
4506   dm->ops->refinehierarchy           = DMRefineHierarchy_Plex;
4507   dm->ops->coarsenhierarchy          = DMCoarsenHierarchy_Plex;
4508   dm->ops->extrude                   = DMExtrude_Plex;
4509   dm->ops->globaltolocalbegin        = NULL;
4510   dm->ops->globaltolocalend          = NULL;
4511   dm->ops->localtoglobalbegin        = NULL;
4512   dm->ops->localtoglobalend          = NULL;
4513   dm->ops->destroy                   = DMDestroy_Plex;
4514   dm->ops->createsubdm               = DMCreateSubDM_Plex;
4515   dm->ops->createsuperdm             = DMCreateSuperDM_Plex;
4516   dm->ops->getdimpoints              = DMGetDimPoints_Plex;
4517   dm->ops->locatepoints              = DMLocatePoints_Plex;
4518   dm->ops->projectfunctionlocal      = DMProjectFunctionLocal_Plex;
4519   dm->ops->projectfunctionlabellocal = DMProjectFunctionLabelLocal_Plex;
4520   dm->ops->projectfieldlocal         = DMProjectFieldLocal_Plex;
4521   dm->ops->projectfieldlabellocal    = DMProjectFieldLabelLocal_Plex;
4522   dm->ops->projectbdfieldlabellocal  = DMProjectBdFieldLabelLocal_Plex;
4523   dm->ops->computel2diff             = DMComputeL2Diff_Plex;
4524   dm->ops->computel2gradientdiff     = DMComputeL2GradientDiff_Plex;
4525   dm->ops->computel2fielddiff        = DMComputeL2FieldDiff_Plex;
4526   dm->ops->getneighbors              = DMGetNeighbors_Plex;
4527   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertBoundaryValues_C", DMPlexInsertBoundaryValues_Plex));
4528   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertTimeDerviativeBoundaryValues_C", DMPlexInsertTimeDerivativeBoundaryValues_Plex));
4529   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMSetUpGLVisViewer_C", DMSetUpGLVisViewer_Plex));
4530   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMCreateNeumannOverlap_C", DMCreateNeumannOverlap_Plex));
4531   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4532   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeGetDefault_C", DMPlexDistributeGetDefault_Plex));
4533   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeSetDefault_C", DMPlexDistributeSetDefault_Plex));
4534   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderGetDefault_C", DMPlexReorderGetDefault_Plex));
4535   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderSetDefault_C", DMPlexReorderSetDefault_Plex));
4536   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMInterpolateSolution_C", DMInterpolateSolution_Plex));
4537   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4538   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetOverlap_C", DMPlexSetOverlap_Plex));
4539   PetscFunctionReturn(PETSC_SUCCESS);
4540 }
4541 
4542 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
4543 {
4544   DM_Plex *mesh = (DM_Plex *)dm->data;
4545   PetscSF  face_sf;
4546 
4547   PetscFunctionBegin;
4548   mesh->refct++;
4549   (*newdm)->data = mesh;
4550   PetscCall(DMPlexGetIsoperiodicFaceSF(dm, &face_sf));
4551   PetscCall(DMPlexSetIsoperiodicFaceSF(*newdm, face_sf));
4552   PetscCall(PetscObjectChangeTypeName((PetscObject)*newdm, DMPLEX));
4553   PetscCall(DMInitialize_Plex(*newdm));
4554   PetscFunctionReturn(PETSC_SUCCESS);
4555 }
4556 
4557 /*MC
4558   DMPLEX = "plex" - A `DM` object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
4559                     In the local representation, Vecs contain all unknowns in the interior and shared boundary. This is
4560                     specified by a PetscSection object. Ownership in the global representation is determined by
4561                     ownership of the underlying `DMPLEX` points. This is specified by another `PetscSection` object.
4562 
4563   Options Database Keys:
4564 + -dm_refine_pre                     - Refine mesh before distribution
4565 + -dm_refine_uniform_pre             - Choose uniform or generator-based refinement
4566 + -dm_refine_volume_limit_pre        - Cell volume limit after pre-refinement using generator
4567 . -dm_distribute                     - Distribute mesh across processes
4568 . -dm_distribute_overlap             - Number of cells to overlap for distribution
4569 . -dm_refine                         - Refine mesh after distribution
4570 . -dm_plex_hash_location             - Use grid hashing for point location
4571 . -dm_plex_hash_box_faces <n,m,p>    - The number of divisions in each direction of the grid hash
4572 . -dm_plex_partition_balance         - Attempt to evenly divide points on partition boundary between processes
4573 . -dm_plex_remesh_bd                 - Allow changes to the boundary on remeshing
4574 . -dm_plex_max_projection_height     - Maximum mesh point height used to project locally
4575 . -dm_plex_regular_refinement        - Use special nested projection algorithm for regular refinement
4576 . -dm_plex_check_all                 - Perform all shecks below
4577 . -dm_plex_check_symmetry            - Check that the adjacency information in the mesh is symmetric
4578 . -dm_plex_check_skeleton <celltype> - Check that each cell has the correct number of vertices
4579 . -dm_plex_check_faces <celltype>    - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
4580 . -dm_plex_check_geometry            - Check that cells have positive volume
4581 . -dm_view :mesh.tex:ascii_latex     - View the mesh in LaTeX/TikZ
4582 . -dm_plex_view_scale <num>          - Scale the TikZ
4583 - -dm_plex_print_fem <num>           - View FEM assembly information, such as element vectors and matrices
4584 
4585   Level: intermediate
4586 
4587 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`, `PetscSection`
4588 M*/
4589 
4590 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
4591 {
4592   DM_Plex *mesh;
4593   PetscInt unit;
4594 
4595   PetscFunctionBegin;
4596   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4597   PetscCall(PetscNew(&mesh));
4598   dm->data = mesh;
4599 
4600   mesh->refct = 1;
4601   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection));
4602   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection));
4603   mesh->refinementUniform      = PETSC_TRUE;
4604   mesh->refinementLimit        = -1.0;
4605   mesh->distDefault            = PETSC_TRUE;
4606   mesh->reorderDefault         = DMPLEX_REORDER_DEFAULT_NOTSET;
4607   mesh->distributionName       = NULL;
4608   mesh->interpolated           = DMPLEX_INTERPOLATED_INVALID;
4609   mesh->interpolatedCollective = DMPLEX_INTERPOLATED_INVALID;
4610 
4611   PetscCall(PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner));
4612   mesh->remeshBd = PETSC_FALSE;
4613 
4614   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
4615 
4616   mesh->depthState    = -1;
4617   mesh->celltypeState = -1;
4618   mesh->printTol      = 1.0e-10;
4619 
4620   PetscCall(DMInitialize_Plex(dm));
4621   PetscFunctionReturn(PETSC_SUCCESS);
4622 }
4623 
4624 /*@
4625   DMPlexCreate - Creates a `DMPLEX` object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
4626 
4627   Collective
4628 
4629   Input Parameter:
4630 . comm - The communicator for the `DMPLEX` object
4631 
4632   Output Parameter:
4633 . mesh  - The `DMPLEX` object
4634 
4635   Level: beginner
4636 
4637 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`
4638 @*/
4639 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
4640 {
4641   PetscFunctionBegin;
4642   PetscValidPointer(mesh, 2);
4643   PetscCall(DMCreate(comm, mesh));
4644   PetscCall(DMSetType(*mesh, DMPLEX));
4645   PetscFunctionReturn(PETSC_SUCCESS);
4646 }
4647 
4648 /*@C
4649   DMPlexBuildFromCellListParallel - Build distributed `DMPLEX` topology from a list of vertices for each cell (common mesh generator output)
4650 
4651   Collective on dm
4652 
4653   Input Parameters:
4654 + dm - The `DM`
4655 . numCells - The number of cells owned by this process
4656 . numVertices - The number of vertices to be owned by this process, or `PETSC_DECIDE`
4657 . NVertices - The global number of vertices, or `PETSC_DETERMINE`
4658 . numCorners - The number of vertices for each cell
4659 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4660 
4661   Output Parameters:
4662 + vertexSF - (Optional) `PetscSF` describing complete vertex ownership
4663 - verticesAdjSaved - (Optional) vertex adjacency array
4664 
4665   Level: advanced
4666 
4667   Notes:
4668   Two triangles sharing a face
4669 .vb
4670 
4671         2
4672       / | \
4673      /  |  \
4674     /   |   \
4675    0  0 | 1  3
4676     \   |   /
4677      \  |  /
4678       \ | /
4679         1
4680 .ve
4681 would have input
4682 .vb
4683   numCells = 2, numVertices = 4
4684   cells = [0 1 2  1 3 2]
4685 .ve
4686 which would result in the `DMPLEX`
4687 .vb
4688 
4689         4
4690       / | \
4691      /  |  \
4692     /   |   \
4693    2  0 | 1  5
4694     \   |   /
4695      \  |  /
4696       \ | /
4697         3
4698 .ve
4699 
4700   Vertices are implicitly numbered consecutively 0,...,NVertices.
4701   Each rank owns a chunk of numVertices consecutive vertices.
4702   If numVertices is `PETSC_DECIDE`, PETSc will distribute them as evenly as possible using PetscLayout.
4703   If NVertices is `PETSC_DETERMINE` and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
4704   If only NVertices is `PETSC_DETERMINE`, it is computed as the sum of numVertices over all ranks.
4705 
4706   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
4707 
4708   Fortran Note:
4709   Not currently supported in Fortran.
4710 
4711 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildCoordinatesFromCellListParallel()`,
4712           `PetscSF`
4713 @*/
4714 PetscErrorCode DMPlexBuildFromCellListParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, const PetscInt cells[], PetscSF *vertexSF, PetscInt **verticesAdjSaved)
4715 {
4716   PetscSF     sfPoint;
4717   PetscLayout layout;
4718   PetscInt    numVerticesAdj, *verticesAdj, *cones, c, p;
4719 
4720   PetscFunctionBegin;
4721   PetscValidLogicalCollectiveInt(dm, NVertices, 4);
4722   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4723   /* Get/check global number of vertices */
4724   {
4725     PetscInt       NVerticesInCells, i;
4726     const PetscInt len = numCells * numCorners;
4727 
4728     /* NVerticesInCells = max(cells) + 1 */
4729     NVerticesInCells = PETSC_MIN_INT;
4730     for (i = 0; i < len; i++)
4731       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4732     ++NVerticesInCells;
4733     PetscCallMPI(MPI_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
4734 
4735     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
4736     else
4737       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
4738   }
4739   /* Count locally unique vertices */
4740   {
4741     PetscHSetI vhash;
4742     PetscInt   off = 0;
4743 
4744     PetscCall(PetscHSetICreate(&vhash));
4745     for (c = 0; c < numCells; ++c) {
4746       for (p = 0; p < numCorners; ++p) PetscCall(PetscHSetIAdd(vhash, cells[c * numCorners + p]));
4747     }
4748     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
4749     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
4750     else verticesAdj = *verticesAdjSaved;
4751     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
4752     PetscCall(PetscHSetIDestroy(&vhash));
4753     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
4754   }
4755   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
4756   /* Create cones */
4757   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
4758   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
4759   PetscCall(DMSetUp(dm));
4760   PetscCall(DMPlexGetCones(dm, &cones));
4761   for (c = 0; c < numCells; ++c) {
4762     for (p = 0; p < numCorners; ++p) {
4763       const PetscInt gv = cells[c * numCorners + p];
4764       PetscInt       lv;
4765 
4766       /* Positions within verticesAdj form 0-based local vertex numbering;
4767          we need to shift it by numCells to get correct DAG points (cells go first) */
4768       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
4769       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
4770       cones[c * numCorners + p] = lv + numCells;
4771     }
4772   }
4773   /* Build point sf */
4774   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
4775   PetscCall(PetscLayoutSetSize(layout, NVertices));
4776   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
4777   PetscCall(PetscLayoutSetBlockSize(layout, 1));
4778   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
4779   PetscCall(PetscLayoutDestroy(&layout));
4780   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
4781   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
4782   if (dm->sf) {
4783     const char *prefix;
4784 
4785     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
4786     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
4787   }
4788   PetscCall(DMSetPointSF(dm, sfPoint));
4789   PetscCall(PetscSFDestroy(&sfPoint));
4790   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)(*vertexSF), "Vertex Ownership SF"));
4791   /* Fill in the rest of the topology structure */
4792   PetscCall(DMPlexSymmetrize(dm));
4793   PetscCall(DMPlexStratify(dm));
4794   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4795   PetscFunctionReturn(PETSC_SUCCESS);
4796 }
4797 
4798 /*@C
4799   DMPlexBuildCoordinatesFromCellListParallel - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4800 
4801   Collective on dm
4802 
4803   Input Parameters:
4804 + dm - The `DM`
4805 . spaceDim - The spatial dimension used for coordinates
4806 . sfVert - `PetscSF` describing complete vertex ownership
4807 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4808 
4809   Level: advanced
4810 
4811   Fortran Note:
4812   Not currently supported in Fortran.
4813 
4814 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellListParallel()`
4815 @*/
4816 PetscErrorCode DMPlexBuildCoordinatesFromCellListParallel(DM dm, PetscInt spaceDim, PetscSF sfVert, const PetscReal vertexCoords[])
4817 {
4818   PetscSection coordSection;
4819   Vec          coordinates;
4820   PetscScalar *coords;
4821   PetscInt     numVertices, numVerticesAdj, coordSize, v, vStart, vEnd;
4822 
4823   PetscFunctionBegin;
4824   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4825   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
4826   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4827   PetscCall(DMSetCoordinateDim(dm, spaceDim));
4828   PetscCall(PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL));
4829   PetscCheck(vEnd - vStart == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Supplied sfVert has wrong number of leaves = %" PetscInt_FMT " != %" PetscInt_FMT " = vEnd - vStart", numVerticesAdj, vEnd - vStart);
4830   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4831   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4832   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
4833   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
4834   for (v = vStart; v < vEnd; ++v) {
4835     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
4836     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
4837   }
4838   PetscCall(PetscSectionSetUp(coordSection));
4839   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4840   PetscCall(VecCreate(PetscObjectComm((PetscObject)dm), &coordinates));
4841   PetscCall(VecSetBlockSize(coordinates, spaceDim));
4842   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4843   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4844   PetscCall(VecSetType(coordinates, VECSTANDARD));
4845   PetscCall(VecGetArray(coordinates, &coords));
4846   {
4847     MPI_Datatype coordtype;
4848 
4849     /* Need a temp buffer for coords if we have complex/single */
4850     PetscCallMPI(MPI_Type_contiguous(spaceDim, MPIU_SCALAR, &coordtype));
4851     PetscCallMPI(MPI_Type_commit(&coordtype));
4852 #if defined(PETSC_USE_COMPLEX)
4853     {
4854       PetscScalar *svertexCoords;
4855       PetscInt     i;
4856       PetscCall(PetscMalloc1(numVertices * spaceDim, &svertexCoords));
4857       for (i = 0; i < numVertices * spaceDim; i++) svertexCoords[i] = vertexCoords[i];
4858       PetscCall(PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4859       PetscCall(PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4860       PetscCall(PetscFree(svertexCoords));
4861     }
4862 #else
4863     PetscCall(PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4864     PetscCall(PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4865 #endif
4866     PetscCallMPI(MPI_Type_free(&coordtype));
4867   }
4868   PetscCall(VecRestoreArray(coordinates, &coords));
4869   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4870   PetscCall(VecDestroy(&coordinates));
4871   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4872   PetscFunctionReturn(PETSC_SUCCESS);
4873 }
4874 
4875 /*@
4876   DMPlexCreateFromCellListParallelPetsc - Create distributed `DMPLEX` from a list of vertices for each cell (common mesh generator output)
4877 
4878   Collective
4879 
4880   Input Parameters:
4881 + comm - The communicator
4882 . dim - The topological dimension of the mesh
4883 . numCells - The number of cells owned by this process
4884 . numVertices - The number of vertices owned by this process, or `PETSC_DECIDE`
4885 . NVertices - The global number of vertices, or `PETSC_DECIDE`
4886 . numCorners - The number of vertices for each cell
4887 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4888 . cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4889 . spaceDim - The spatial dimension used for coordinates
4890 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4891 
4892   Output Parameters:
4893 + dm - The `DM`
4894 . vertexSF - (Optional) `PetscSF` describing complete vertex ownership
4895 - verticesAdjSaved - (Optional) vertex adjacency array
4896 
4897   Level: intermediate
4898 
4899   Notes:
4900   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`,
4901   `DMPlexBuildFromCellListParallel()`, `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellListParallel()`
4902 
4903   See `DMPlexBuildFromCellListParallel()` for an example and details about the topology-related parameters.
4904 
4905   See `DMPlexBuildCoordinatesFromCellListParallel()` for details about the geometry-related parameters.
4906 
4907 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
4908 @*/
4909 PetscErrorCode DMPlexCreateFromCellListParallelPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, PetscInt **verticesAdj, DM *dm)
4910 {
4911   PetscSF sfVert;
4912 
4913   PetscFunctionBegin;
4914   PetscCall(DMCreate(comm, dm));
4915   PetscCall(DMSetType(*dm, DMPLEX));
4916   PetscValidLogicalCollectiveInt(*dm, dim, 2);
4917   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
4918   PetscCall(DMSetDimension(*dm, dim));
4919   PetscCall(DMPlexBuildFromCellListParallel(*dm, numCells, numVertices, NVertices, numCorners, cells, &sfVert, verticesAdj));
4920   if (interpolate) {
4921     DM idm;
4922 
4923     PetscCall(DMPlexInterpolate(*dm, &idm));
4924     PetscCall(DMDestroy(dm));
4925     *dm = idm;
4926   }
4927   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
4928   if (vertexSF) *vertexSF = sfVert;
4929   else PetscCall(PetscSFDestroy(&sfVert));
4930   PetscFunctionReturn(PETSC_SUCCESS);
4931 }
4932 
4933 /*@C
4934   DMPlexBuildFromCellList - Build `DMPLEX` topology from a list of vertices for each cell (common mesh generator output)
4935 
4936   Collective on dm
4937 
4938   Input Parameters:
4939 + dm - The `DM`
4940 . numCells - The number of cells owned by this process
4941 . numVertices - The number of vertices owned by this process, or `PETSC_DETERMINE`
4942 . numCorners - The number of vertices for each cell
4943 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4944 
4945   Level: advanced
4946 
4947   Notes:
4948   Two triangles sharing a face
4949 .vb
4950 
4951         2
4952       / | \
4953      /  |  \
4954     /   |   \
4955    0  0 | 1  3
4956     \   |   /
4957      \  |  /
4958       \ | /
4959         1
4960 .ve
4961 would have input
4962 .vb
4963   numCells = 2, numVertices = 4
4964   cells = [0 1 2  1 3 2]
4965 .ve
4966 which would result in the `DMPLEX`
4967 .vb
4968 
4969         4
4970       / | \
4971      /  |  \
4972     /   |   \
4973    2  0 | 1  5
4974     \   |   /
4975      \  |  /
4976       \ | /
4977         3
4978 .ve
4979 
4980   If numVertices is `PETSC_DETERMINE`, it is computed by PETSc as the maximum vertex index in cells + 1.
4981 
4982   Not currently supported in Fortran.
4983 
4984 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListPetsc()`
4985 @*/
4986 PetscErrorCode DMPlexBuildFromCellList(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const PetscInt cells[])
4987 {
4988   PetscInt *cones, c, p, dim;
4989 
4990   PetscFunctionBegin;
4991   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4992   PetscCall(DMGetDimension(dm, &dim));
4993   /* Get/check global number of vertices */
4994   {
4995     PetscInt       NVerticesInCells, i;
4996     const PetscInt len = numCells * numCorners;
4997 
4998     /* NVerticesInCells = max(cells) + 1 */
4999     NVerticesInCells = PETSC_MIN_INT;
5000     for (i = 0; i < len; i++)
5001       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
5002     ++NVerticesInCells;
5003 
5004     if (numVertices == PETSC_DECIDE) numVertices = NVerticesInCells;
5005     else
5006       PetscCheck(numVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, numVertices, NVerticesInCells);
5007   }
5008   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
5009   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
5010   PetscCall(DMSetUp(dm));
5011   PetscCall(DMPlexGetCones(dm, &cones));
5012   for (c = 0; c < numCells; ++c) {
5013     for (p = 0; p < numCorners; ++p) cones[c * numCorners + p] = cells[c * numCorners + p] + numCells;
5014   }
5015   PetscCall(DMPlexSymmetrize(dm));
5016   PetscCall(DMPlexStratify(dm));
5017   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5018   PetscFunctionReturn(PETSC_SUCCESS);
5019 }
5020 
5021 /*@C
5022   DMPlexBuildCoordinatesFromCellList - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
5023 
5024   Collective on dm
5025 
5026   Input Parameters:
5027 + dm - The `DM`
5028 . spaceDim - The spatial dimension used for coordinates
5029 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
5030 
5031   Level: advanced
5032 
5033   Fortran Note:
5034   Not currently supported in Fortran.
5035 
5036 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellList()`
5037 @*/
5038 PetscErrorCode DMPlexBuildCoordinatesFromCellList(DM dm, PetscInt spaceDim, const PetscReal vertexCoords[])
5039 {
5040   PetscSection coordSection;
5041   Vec          coordinates;
5042   DM           cdm;
5043   PetscScalar *coords;
5044   PetscInt     v, vStart, vEnd, d;
5045 
5046   PetscFunctionBegin;
5047   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
5048   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
5049   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
5050   PetscCall(DMSetCoordinateDim(dm, spaceDim));
5051   PetscCall(DMGetCoordinateSection(dm, &coordSection));
5052   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5053   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
5054   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
5055   for (v = vStart; v < vEnd; ++v) {
5056     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
5057     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
5058   }
5059   PetscCall(PetscSectionSetUp(coordSection));
5060 
5061   PetscCall(DMGetCoordinateDM(dm, &cdm));
5062   PetscCall(DMCreateLocalVector(cdm, &coordinates));
5063   PetscCall(VecSetBlockSize(coordinates, spaceDim));
5064   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5065   PetscCall(VecGetArrayWrite(coordinates, &coords));
5066   for (v = 0; v < vEnd - vStart; ++v) {
5067     for (d = 0; d < spaceDim; ++d) coords[v * spaceDim + d] = vertexCoords[v * spaceDim + d];
5068   }
5069   PetscCall(VecRestoreArrayWrite(coordinates, &coords));
5070   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
5071   PetscCall(VecDestroy(&coordinates));
5072   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
5073   PetscFunctionReturn(PETSC_SUCCESS);
5074 }
5075 
5076 /*@
5077   DMPlexCreateFromCellListPetsc - Create `DMPLEX` from a list of vertices for each cell (common mesh generator output), but only process 0 takes in the input
5078 
5079   Collective
5080 
5081   Input Parameters:
5082 + comm - The communicator
5083 . dim - The topological dimension of the mesh
5084 . numCells - The number of cells, only on process 0
5085 . numVertices - The number of vertices owned by this process, or `PETSC_DECIDE`, only on process 0
5086 . numCorners - The number of vertices for each cell, only on process 0
5087 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
5088 . cells - An array of numCells*numCorners numbers, the vertices for each cell, only on process 0
5089 . spaceDim - The spatial dimension used for coordinates
5090 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex, only on process 0
5091 
5092   Output Parameter:
5093 . dm - The `DM`, which only has points on process 0
5094 
5095   Level: intermediate
5096 
5097   Notes:
5098   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`, `DMPlexBuildFromCellList()`,
5099   `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellList()`
5100 
5101   See `DMPlexBuildFromCellList()` for an example and details about the topology-related parameters.
5102   See `DMPlexBuildCoordinatesFromCellList()` for details about the geometry-related parameters.
5103   See `DMPlexCreateFromCellListParallelPetsc()` for parallel input
5104 
5105 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellList()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
5106 @*/
5107 PetscErrorCode DMPlexCreateFromCellListPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], DM *dm)
5108 {
5109   PetscMPIInt rank;
5110 
5111   PetscFunctionBegin;
5112   PetscCheck(dim, comm, PETSC_ERR_ARG_OUTOFRANGE, "This is not appropriate for 0-dimensional meshes. Consider either creating the DM using DMPlexCreateFromDAG(), by hand, or using DMSwarm.");
5113   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5114   PetscCall(DMCreate(comm, dm));
5115   PetscCall(DMSetType(*dm, DMPLEX));
5116   PetscCall(DMSetDimension(*dm, dim));
5117   if (rank == 0) PetscCall(DMPlexBuildFromCellList(*dm, numCells, numVertices, numCorners, cells));
5118   else PetscCall(DMPlexBuildFromCellList(*dm, 0, 0, 0, NULL));
5119   if (interpolate) {
5120     DM idm;
5121 
5122     PetscCall(DMPlexInterpolate(*dm, &idm));
5123     PetscCall(DMDestroy(dm));
5124     *dm = idm;
5125   }
5126   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, vertexCoords));
5127   else PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, NULL));
5128   PetscFunctionReturn(PETSC_SUCCESS);
5129 }
5130 
5131 /*@
5132   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a DM
5133 
5134   Input Parameters:
5135 + dm - The empty DM object, usually from DMCreate() and DMSetDimension()
5136 . depth - The depth of the DAG
5137 . numPoints - Array of size depth + 1 containing the number of points at each depth
5138 . coneSize - The cone size of each point
5139 . cones - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
5140 . coneOrientations - The orientation of each cone point
5141 - vertexCoords - An array of numPoints[0]*spacedim numbers representing the coordinates of each vertex, with spacedim the value set via DMSetCoordinateDim()
5142 
5143   Output Parameter:
5144 . dm - The DM
5145 
5146   Note:
5147   Two triangles sharing a face would have input
5148 .vb
5149   depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
5150   cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
5151  vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
5152 .ve
5153 which would result in the DMPlex
5154 .vb
5155         4
5156       / | \
5157      /  |  \
5158     /   |   \
5159    2  0 | 1  5
5160     \   |   /
5161      \  |  /
5162       \ | /
5163         3
5164 .ve
5165  Notice that all points are numbered consecutively, unlike `DMPlexCreateFromCellListPetsc()`
5166 
5167   Level: advanced
5168 
5169 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
5170 @*/
5171 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
5172 {
5173   Vec          coordinates;
5174   PetscSection coordSection;
5175   PetscScalar *coords;
5176   PetscInt     coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
5177 
5178   PetscFunctionBegin;
5179   PetscCall(DMGetDimension(dm, &dim));
5180   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
5181   PetscCheck(dimEmbed >= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Embedding dimension %" PetscInt_FMT " cannot be less than intrinsic dimension %" PetscInt_FMT, dimEmbed, dim);
5182   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
5183   PetscCall(DMPlexSetChart(dm, pStart, pEnd));
5184   for (p = pStart; p < pEnd; ++p) {
5185     PetscCall(DMPlexSetConeSize(dm, p, coneSize[p - pStart]));
5186     if (firstVertex < 0 && !coneSize[p - pStart]) firstVertex = p - pStart;
5187   }
5188   PetscCheck(firstVertex >= 0 || !numPoints[0], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Expected %" PetscInt_FMT " vertices but could not find any", numPoints[0]);
5189   PetscCall(DMSetUp(dm)); /* Allocate space for cones */
5190   for (p = pStart, off = 0; p < pEnd; off += coneSize[p - pStart], ++p) {
5191     PetscCall(DMPlexSetCone(dm, p, &cones[off]));
5192     PetscCall(DMPlexSetConeOrientation(dm, p, &coneOrientations[off]));
5193   }
5194   PetscCall(DMPlexSymmetrize(dm));
5195   PetscCall(DMPlexStratify(dm));
5196   /* Build coordinates */
5197   PetscCall(DMGetCoordinateSection(dm, &coordSection));
5198   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5199   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dimEmbed));
5200   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numPoints[0]));
5201   for (v = firstVertex; v < firstVertex + numPoints[0]; ++v) {
5202     PetscCall(PetscSectionSetDof(coordSection, v, dimEmbed));
5203     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed));
5204   }
5205   PetscCall(PetscSectionSetUp(coordSection));
5206   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
5207   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
5208   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5209   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
5210   PetscCall(VecSetBlockSize(coordinates, dimEmbed));
5211   PetscCall(VecSetType(coordinates, VECSTANDARD));
5212   if (vertexCoords) {
5213     PetscCall(VecGetArray(coordinates, &coords));
5214     for (v = 0; v < numPoints[0]; ++v) {
5215       PetscInt off;
5216 
5217       PetscCall(PetscSectionGetOffset(coordSection, v + firstVertex, &off));
5218       for (d = 0; d < dimEmbed; ++d) coords[off + d] = vertexCoords[v * dimEmbed + d];
5219     }
5220   }
5221   PetscCall(VecRestoreArray(coordinates, &coords));
5222   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
5223   PetscCall(VecDestroy(&coordinates));
5224   PetscFunctionReturn(PETSC_SUCCESS);
5225 }
5226 
5227 /*@C
5228   DMPlexCreateCellVertexFromFile - Create a `DMPLEX` mesh from a simple cell-vertex file.
5229 
5230   Collective
5231 
5232 + comm        - The MPI communicator
5233 . filename    - Name of the .dat file
5234 - interpolate - Create faces and edges in the mesh
5235 
5236   Output Parameter:
5237 . dm  - The `DM` object representing the mesh
5238 
5239   Level: beginner
5240 
5241   Note:
5242   The format is the simplest possible:
5243 .vb
5244   Ne
5245   v0 v1 ... vk
5246   Nv
5247   x y z marker
5248 .ve
5249 
5250   Developer Note:
5251   Should use a `PetscViewer` not a filename
5252 
5253 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromFile()`, `DMPlexCreateMedFromFile()`, `DMPlexCreateGmsh()`, `DMPlexCreate()`
5254 @*/
5255 static PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
5256 {
5257   DMLabel      marker;
5258   PetscViewer  viewer;
5259   Vec          coordinates;
5260   PetscSection coordSection;
5261   PetscScalar *coords;
5262   char         line[PETSC_MAX_PATH_LEN];
5263   PetscInt     dim = 3, cdim = 3, coordSize, v, c, d;
5264   PetscMPIInt  rank;
5265   int          snum, Nv, Nc, Ncn, Nl;
5266 
5267   PetscFunctionBegin;
5268   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5269   PetscCall(PetscViewerCreate(comm, &viewer));
5270   PetscCall(PetscViewerSetType(viewer, PETSCVIEWERASCII));
5271   PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
5272   PetscCall(PetscViewerFileSetName(viewer, filename));
5273   if (rank == 0) {
5274     PetscCall(PetscViewerRead(viewer, line, 4, NULL, PETSC_STRING));
5275     snum = sscanf(line, "%d %d %d %d", &Nc, &Nv, &Ncn, &Nl);
5276     PetscCheck(snum == 4, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5277   } else {
5278     Nc = Nv = Ncn = Nl = 0;
5279   }
5280   PetscCall(DMCreate(comm, dm));
5281   PetscCall(DMSetType(*dm, DMPLEX));
5282   PetscCall(DMPlexSetChart(*dm, 0, Nc + Nv));
5283   PetscCall(DMSetDimension(*dm, dim));
5284   PetscCall(DMSetCoordinateDim(*dm, cdim));
5285   /* Read topology */
5286   if (rank == 0) {
5287     char     format[PETSC_MAX_PATH_LEN];
5288     PetscInt cone[8];
5289     int      vbuf[8], v;
5290 
5291     for (c = 0; c < Ncn; ++c) {
5292       format[c * 3 + 0] = '%';
5293       format[c * 3 + 1] = 'd';
5294       format[c * 3 + 2] = ' ';
5295     }
5296     format[Ncn * 3 - 1] = '\0';
5297     for (c = 0; c < Nc; ++c) PetscCall(DMPlexSetConeSize(*dm, c, Ncn));
5298     PetscCall(DMSetUp(*dm));
5299     for (c = 0; c < Nc; ++c) {
5300       PetscCall(PetscViewerRead(viewer, line, Ncn, NULL, PETSC_STRING));
5301       switch (Ncn) {
5302       case 2:
5303         snum = sscanf(line, format, &vbuf[0], &vbuf[1]);
5304         break;
5305       case 3:
5306         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2]);
5307         break;
5308       case 4:
5309         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3]);
5310         break;
5311       case 6:
5312         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5]);
5313         break;
5314       case 8:
5315         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);
5316         break;
5317       default:
5318         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No cell shape with %d vertices", Ncn);
5319       }
5320       PetscCheck(snum == Ncn, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5321       for (v = 0; v < Ncn; ++v) cone[v] = vbuf[v] + Nc;
5322       /* Hexahedra are inverted */
5323       if (Ncn == 8) {
5324         PetscInt tmp = cone[1];
5325         cone[1]      = cone[3];
5326         cone[3]      = tmp;
5327       }
5328       PetscCall(DMPlexSetCone(*dm, c, cone));
5329     }
5330   }
5331   PetscCall(DMPlexSymmetrize(*dm));
5332   PetscCall(DMPlexStratify(*dm));
5333   /* Read coordinates */
5334   PetscCall(DMGetCoordinateSection(*dm, &coordSection));
5335   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5336   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
5337   PetscCall(PetscSectionSetChart(coordSection, Nc, Nc + Nv));
5338   for (v = Nc; v < Nc + Nv; ++v) {
5339     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
5340     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
5341   }
5342   PetscCall(PetscSectionSetUp(coordSection));
5343   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
5344   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
5345   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5346   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
5347   PetscCall(VecSetBlockSize(coordinates, cdim));
5348   PetscCall(VecSetType(coordinates, VECSTANDARD));
5349   PetscCall(VecGetArray(coordinates, &coords));
5350   if (rank == 0) {
5351     char   format[PETSC_MAX_PATH_LEN];
5352     double x[3];
5353     int    l, val[3];
5354 
5355     if (Nl) {
5356       for (l = 0; l < Nl; ++l) {
5357         format[l * 3 + 0] = '%';
5358         format[l * 3 + 1] = 'd';
5359         format[l * 3 + 2] = ' ';
5360       }
5361       format[Nl * 3 - 1] = '\0';
5362       PetscCall(DMCreateLabel(*dm, "marker"));
5363       PetscCall(DMGetLabel(*dm, "marker", &marker));
5364     }
5365     for (v = 0; v < Nv; ++v) {
5366       PetscCall(PetscViewerRead(viewer, line, 3 + Nl, NULL, PETSC_STRING));
5367       snum = sscanf(line, "%lg %lg %lg", &x[0], &x[1], &x[2]);
5368       PetscCheck(snum == 3, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5369       switch (Nl) {
5370       case 0:
5371         snum = 0;
5372         break;
5373       case 1:
5374         snum = sscanf(line, format, &val[0]);
5375         break;
5376       case 2:
5377         snum = sscanf(line, format, &val[0], &val[1]);
5378         break;
5379       case 3:
5380         snum = sscanf(line, format, &val[0], &val[1], &val[2]);
5381         break;
5382       default:
5383         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Request support for %d labels", Nl);
5384       }
5385       PetscCheck(snum == Nl, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5386       for (d = 0; d < cdim; ++d) coords[v * cdim + d] = x[d];
5387       for (l = 0; l < Nl; ++l) PetscCall(DMLabelSetValue(marker, v + Nc, val[l]));
5388     }
5389   }
5390   PetscCall(VecRestoreArray(coordinates, &coords));
5391   PetscCall(DMSetCoordinatesLocal(*dm, coordinates));
5392   PetscCall(VecDestroy(&coordinates));
5393   PetscCall(PetscViewerDestroy(&viewer));
5394   if (interpolate) {
5395     DM      idm;
5396     DMLabel bdlabel;
5397 
5398     PetscCall(DMPlexInterpolate(*dm, &idm));
5399     PetscCall(DMDestroy(dm));
5400     *dm = idm;
5401 
5402     if (!Nl) {
5403       PetscCall(DMCreateLabel(*dm, "marker"));
5404       PetscCall(DMGetLabel(*dm, "marker", &bdlabel));
5405       PetscCall(DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel));
5406       PetscCall(DMPlexLabelComplete(*dm, bdlabel));
5407     }
5408   }
5409   PetscFunctionReturn(PETSC_SUCCESS);
5410 }
5411 
5412 /*@C
5413   DMPlexCreateFromFile - This takes a filename and produces a `DM`
5414 
5415   Collective
5416 
5417   Input Parameters:
5418 + comm - The communicator
5419 . filename - A file name
5420 . plexname - The object name of the resulting `DM`, also used for intra-datafile lookup by some formats
5421 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
5422 
5423   Output Parameter:
5424 . dm - The `DM`
5425 
5426   Options Database Key:
5427 . -dm_plex_create_from_hdf5_xdmf - use the `PETSC_VIEWER_HDF5_XDMF` format for reading HDF5
5428 
5429   Use -dm_plex_create_ prefix to pass options to the internal PetscViewer, e.g.
5430 $ -dm_plex_create_viewer_hdf5_collective
5431 
5432   Level: beginner
5433 
5434   Notes:
5435   Using `PETSCVIEWERHDF5` type with `PETSC_VIEWER_HDF5_PETSC` format, one can save multiple `DMPLEX`
5436   meshes in a single HDF5 file. This in turn requires one to name the `DMPLEX` object with `PetscObjectSetName()`
5437   before saving it with `DMView()` and before loading it with `DMLoad()` for identification of the mesh object.
5438   The input parameter name is thus used to name the `DMPLEX` object when `DMPlexCreateFromFile()` internally
5439   calls `DMLoad()`. Currently, name is ignored for other viewer types and/or formats.
5440 
5441 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`, `PetscObjectSetName()`, `DMView()`, `DMLoad()`
5442 @*/
5443 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], const char plexname[], PetscBool interpolate, DM *dm)
5444 {
5445   const char  extGmsh[]      = ".msh";
5446   const char  extGmsh2[]     = ".msh2";
5447   const char  extGmsh4[]     = ".msh4";
5448   const char  extCGNS[]      = ".cgns";
5449   const char  extExodus[]    = ".exo";
5450   const char  extExodus_e[]  = ".e";
5451   const char  extGenesis[]   = ".gen";
5452   const char  extFluent[]    = ".cas";
5453   const char  extHDF5[]      = ".h5";
5454   const char  extMed[]       = ".med";
5455   const char  extPLY[]       = ".ply";
5456   const char  extEGADSLite[] = ".egadslite";
5457   const char  extEGADS[]     = ".egads";
5458   const char  extIGES[]      = ".igs";
5459   const char  extSTEP[]      = ".stp";
5460   const char  extCV[]        = ".dat";
5461   size_t      len;
5462   PetscBool   isGmsh, isGmsh2, isGmsh4, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isMed, isPLY, isEGADSLite, isEGADS, isIGES, isSTEP, isCV;
5463   PetscMPIInt rank;
5464 
5465   PetscFunctionBegin;
5466   PetscValidCharPointer(filename, 2);
5467   if (plexname) PetscValidCharPointer(plexname, 3);
5468   PetscValidPointer(dm, 5);
5469   PetscCall(DMInitializePackage());
5470   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5471   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5472   PetscCall(PetscStrlen(filename, &len));
5473   PetscCheck(len, comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
5474 
5475 #define CheckExtension(extension__, is_extension__) \
5476   do { \
5477     PetscAssert(sizeof(extension__), comm, PETSC_ERR_PLIB, "Zero-size extension: %s", extension__); \
5478     /* don't count the null-terminator at the end */ \
5479     const size_t ext_len = sizeof(extension__) - 1; \
5480     if (len < ext_len) { \
5481       is_extension__ = PETSC_FALSE; \
5482     } else { \
5483       PetscCall(PetscStrncmp(filename + len - ext_len, extension__, ext_len, &is_extension__)); \
5484     } \
5485   } while (0)
5486 
5487   CheckExtension(extGmsh, isGmsh);
5488   CheckExtension(extGmsh2, isGmsh2);
5489   CheckExtension(extGmsh4, isGmsh4);
5490   CheckExtension(extCGNS, isCGNS);
5491   CheckExtension(extExodus, isExodus);
5492   if (!isExodus) CheckExtension(extExodus_e, isExodus);
5493   CheckExtension(extGenesis, isGenesis);
5494   CheckExtension(extFluent, isFluent);
5495   CheckExtension(extHDF5, isHDF5);
5496   CheckExtension(extMed, isMed);
5497   CheckExtension(extPLY, isPLY);
5498   CheckExtension(extEGADSLite, isEGADSLite);
5499   CheckExtension(extEGADS, isEGADS);
5500   CheckExtension(extIGES, isIGES);
5501   CheckExtension(extSTEP, isSTEP);
5502   CheckExtension(extCV, isCV);
5503 
5504 #undef CheckExtension
5505 
5506   if (isGmsh || isGmsh2 || isGmsh4) {
5507     PetscCall(DMPlexCreateGmshFromFile(comm, filename, interpolate, dm));
5508   } else if (isCGNS) {
5509     PetscCall(DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm));
5510   } else if (isExodus || isGenesis) {
5511     PetscCall(DMPlexCreateExodusFromFile(comm, filename, interpolate, dm));
5512   } else if (isFluent) {
5513     PetscCall(DMPlexCreateFluentFromFile(comm, filename, interpolate, dm));
5514   } else if (isHDF5) {
5515     PetscBool   load_hdf5_xdmf = PETSC_FALSE;
5516     PetscViewer viewer;
5517 
5518     /* PETSC_VIEWER_HDF5_XDMF is used if the filename ends with .xdmf.h5, or if -dm_plex_create_from_hdf5_xdmf option is present */
5519     PetscCall(PetscStrncmp(&filename[PetscMax(0, len - 8)], ".xdmf", 5, &load_hdf5_xdmf));
5520     PetscCall(PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &load_hdf5_xdmf, NULL));
5521     PetscCall(PetscViewerCreate(comm, &viewer));
5522     PetscCall(PetscViewerSetType(viewer, PETSCVIEWERHDF5));
5523     PetscCall(PetscViewerSetOptionsPrefix(viewer, "dm_plex_create_"));
5524     PetscCall(PetscViewerSetFromOptions(viewer));
5525     PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
5526     PetscCall(PetscViewerFileSetName(viewer, filename));
5527 
5528     PetscCall(DMCreate(comm, dm));
5529     PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5530     PetscCall(DMSetType(*dm, DMPLEX));
5531     if (load_hdf5_xdmf) PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF));
5532     PetscCall(DMLoad(*dm, viewer));
5533     if (load_hdf5_xdmf) PetscCall(PetscViewerPopFormat(viewer));
5534     PetscCall(PetscViewerDestroy(&viewer));
5535 
5536     if (interpolate) {
5537       DM idm;
5538 
5539       PetscCall(DMPlexInterpolate(*dm, &idm));
5540       PetscCall(DMDestroy(dm));
5541       *dm = idm;
5542     }
5543   } else if (isMed) {
5544     PetscCall(DMPlexCreateMedFromFile(comm, filename, interpolate, dm));
5545   } else if (isPLY) {
5546     PetscCall(DMPlexCreatePLYFromFile(comm, filename, interpolate, dm));
5547   } else if (isEGADSLite || isEGADS || isIGES || isSTEP) {
5548     if (isEGADSLite) PetscCall(DMPlexCreateEGADSLiteFromFile(comm, filename, dm));
5549     else PetscCall(DMPlexCreateEGADSFromFile(comm, filename, dm));
5550     if (!interpolate) {
5551       DM udm;
5552 
5553       PetscCall(DMPlexUninterpolate(*dm, &udm));
5554       PetscCall(DMDestroy(dm));
5555       *dm = udm;
5556     }
5557   } else if (isCV) {
5558     PetscCall(DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm));
5559   } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
5560   PetscCall(PetscStrlen(plexname, &len));
5561   if (len) PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5562   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5563   PetscFunctionReturn(PETSC_SUCCESS);
5564 }
5565 /*@C
5566   DMPlexCreateEphemeral - This takes a `DMPlexTransform` and a base `DMPlex` and produces an ephemeral `DM`, meaning one that is created on the fly in response to queries.
5567 
5568   Input Parameter:
5569 . tr - The `DMPlexTransform`
5570 
5571   Output Parameter:
5572 . dm - The `DM`
5573 
5574   Notes:
5575   An emphemeral mesh is one that is not stored concretely, as in the default Plex implementation, but rather is produced on the fly in response to queries, using information from the transform and the base mesh.
5576 
5577   Level: beginner
5578 
5579 .seealso: `DMPlexCreateFromFile`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
5580 @*/
5581 PetscErrorCode DMPlexCreateEphemeral(DMPlexTransform tr, DM *dm)
5582 {
5583   DM       bdm;
5584   PetscInt Nl;
5585 
5586   PetscFunctionBegin;
5587   PetscCall(DMCreate(PetscObjectComm((PetscObject)tr), dm));
5588   PetscCall(DMSetType(*dm, DMPLEX));
5589   PetscCall(DMSetFromOptions(*dm));
5590 
5591   PetscCall(PetscObjectReference((PetscObject)tr));
5592   PetscCall(DMPlexTransformDestroy(&((DM_Plex *)(*dm)->data)->tr));
5593   ((DM_Plex *)(*dm)->data)->tr = tr;
5594 
5595   PetscCall(DMPlexTransformGetDM(tr, &bdm));
5596   PetscCall(DMGetNumLabels(bdm, &Nl));
5597   for (PetscInt l = 0; l < Nl; ++l) {
5598     DMLabel     label, labelNew;
5599     const char *lname;
5600     PetscBool   isDepth, isCellType;
5601 
5602     PetscCall(DMGetLabelName(bdm, l, &lname));
5603     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
5604     if (isDepth) continue;
5605     PetscCall(PetscStrcmp(lname, "celltype", &isCellType));
5606     if (isCellType) continue;
5607     PetscCall(DMCreateLabel(*dm, lname));
5608     PetscCall(DMGetLabel(bdm, lname, &label));
5609     PetscCall(DMGetLabel(*dm, lname, &labelNew));
5610     PetscCall(DMLabelSetType(labelNew, DMLABELEPHEMERAL));
5611     PetscCall(DMLabelEphemeralSetLabel(labelNew, label));
5612     PetscCall(DMLabelEphemeralSetTransform(labelNew, tr));
5613     PetscCall(DMLabelSetUp(labelNew));
5614   }
5615   PetscFunctionReturn(PETSC_SUCCESS);
5616 }
5617