xref: /petsc/src/dm/impls/plex/plexcreate.c (revision ffc29c3a132cd601571afc997614298c537fee80)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h> /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>
4 #include <petscsf.h>
5 #include <petscdmplextransform.h> /*I   "petscdmplextransform.h"   I*/
6 #include <petscdmlabelephemeral.h>
7 #include <petsc/private/kernels/blockmatmult.h>
8 #include <petsc/private/kernels/blockinvert.h>
9 
10 #ifdef PETSC_HAVE_UNISTD_H
11   #include <unistd.h>
12 #endif
13 #include <errno.h>
14 
15 PetscLogEvent DMPLEX_CreateFromFile, DMPLEX_CreateFromOptions, DMPLEX_BuildFromCellList, DMPLEX_BuildCoordinatesFromCellList;
16 
17 /* External function declarations here */
18 static PetscErrorCode DMInitialize_Plex(DM dm);
19 
20 /* This copies internal things in the Plex structure that we generally want when making a new, related Plex */
21 PetscErrorCode DMPlexCopy_Internal(DM dmin, PetscBool copyPeriodicity, PetscBool copyOverlap, DM dmout)
22 {
23   const PetscReal     *maxCell, *Lstart, *L;
24   VecType              vecType;
25   MatType              matType;
26   PetscBool            dist, useCeed;
27   DMReorderDefaultFlag reorder;
28 
29   PetscFunctionBegin;
30   if (dmin == dmout) PetscFunctionReturn(PETSC_SUCCESS);
31   PetscCall(DMGetVecType(dmin, &vecType));
32   PetscCall(DMSetVecType(dmout, vecType));
33   PetscCall(DMGetMatType(dmin, &matType));
34   PetscCall(DMSetMatType(dmout, matType));
35   if (copyPeriodicity) {
36     PetscCall(DMGetPeriodicity(dmin, &maxCell, &Lstart, &L));
37     PetscCall(DMSetPeriodicity(dmout, maxCell, Lstart, L));
38     PetscCall(DMLocalizeCoordinates(dmout));
39   }
40   PetscCall(DMPlexDistributeGetDefault(dmin, &dist));
41   PetscCall(DMPlexDistributeSetDefault(dmout, dist));
42   PetscCall(DMPlexReorderGetDefault(dmin, &reorder));
43   PetscCall(DMPlexReorderSetDefault(dmout, reorder));
44   PetscCall(DMPlexGetUseCeed(dmin, &useCeed));
45   PetscCall(DMPlexSetUseCeed(dmout, useCeed));
46   ((DM_Plex *)dmout->data)->useHashLocation = ((DM_Plex *)dmin->data)->useHashLocation;
47   ((DM_Plex *)dmout->data)->printSetValues  = ((DM_Plex *)dmin->data)->printSetValues;
48   ((DM_Plex *)dmout->data)->printFEM        = ((DM_Plex *)dmin->data)->printFEM;
49   ((DM_Plex *)dmout->data)->printFVM        = ((DM_Plex *)dmin->data)->printFVM;
50   ((DM_Plex *)dmout->data)->printL2         = ((DM_Plex *)dmin->data)->printL2;
51   ((DM_Plex *)dmout->data)->printLocate     = ((DM_Plex *)dmin->data)->printLocate;
52   ((DM_Plex *)dmout->data)->printProject    = ((DM_Plex *)dmin->data)->printProject;
53   ((DM_Plex *)dmout->data)->printTol        = ((DM_Plex *)dmin->data)->printTol;
54   if (copyOverlap) PetscCall(DMPlexSetOverlap_Plex(dmout, dmin, 0));
55   PetscFunctionReturn(PETSC_SUCCESS);
56 }
57 
58 /* Replace dm with the contents of ndm, and then destroy ndm
59    - Share the DM_Plex structure
60    - Share the coordinates
61    - Share the SF
62 */
63 PetscErrorCode DMPlexReplace_Internal(DM dm, DM *ndm)
64 {
65   PetscSF          sf;
66   DM               dmNew = *ndm, coordDM, coarseDM;
67   Vec              coords;
68   const PetscReal *maxCell, *Lstart, *L;
69   PetscInt         dim, cdim;
70   PetscBool        use_natural;
71 
72   PetscFunctionBegin;
73   if (dm == dmNew) {
74     PetscCall(DMDestroy(ndm));
75     PetscFunctionReturn(PETSC_SUCCESS);
76   }
77   dm->setupcalled = dmNew->setupcalled;
78   if (!dm->hdr.name) {
79     const char *name;
80 
81     PetscCall(PetscObjectGetName((PetscObject)*ndm, &name));
82     PetscCall(PetscObjectSetName((PetscObject)dm, name));
83   }
84   PetscCall(DMGetDimension(dmNew, &dim));
85   PetscCall(DMSetDimension(dm, dim));
86   PetscCall(DMGetCoordinateDim(dmNew, &cdim));
87   PetscCall(DMSetCoordinateDim(dm, cdim));
88   PetscCall(DMGetPointSF(dmNew, &sf));
89   PetscCall(DMSetPointSF(dm, sf));
90   PetscCall(DMGetCoordinateDM(dmNew, &coordDM));
91   PetscCall(DMGetCoordinatesLocal(dmNew, &coords));
92   PetscCall(DMSetCoordinateDM(dm, coordDM));
93   PetscCall(DMSetCoordinatesLocal(dm, coords));
94   PetscCall(DMGetCellCoordinateDM(dmNew, &coordDM));
95   PetscCall(DMGetCellCoordinatesLocal(dmNew, &coords));
96   PetscCall(DMSetCellCoordinateDM(dm, coordDM));
97   PetscCall(DMSetCellCoordinatesLocal(dm, coords));
98   /* Do not want to create the coordinate field if it does not already exist, so do not call DMGetCoordinateField() */
99   PetscCall(DMFieldDestroy(&dm->coordinates[0].field));
100   dm->coordinates[0].field            = dmNew->coordinates[0].field;
101   ((DM_Plex *)dmNew->data)->coordFunc = ((DM_Plex *)dm->data)->coordFunc;
102   PetscCall(DMGetPeriodicity(dmNew, &maxCell, &Lstart, &L));
103   PetscCall(DMSetPeriodicity(dm, maxCell, Lstart, L));
104   PetscCall(DMGetNaturalSF(dmNew, &sf));
105   PetscCall(DMSetNaturalSF(dm, sf));
106   PetscCall(DMGetUseNatural(dmNew, &use_natural));
107   PetscCall(DMSetUseNatural(dm, use_natural));
108   PetscCall(DMDestroy_Plex(dm));
109   PetscCall(DMInitialize_Plex(dm));
110   dm->data = dmNew->data;
111   ((DM_Plex *)dmNew->data)->refct++;
112   {
113     PetscInt       num_face_sfs;
114     const PetscSF *sfs;
115     PetscCall(DMPlexGetIsoperiodicFaceSF(dm, &num_face_sfs, &sfs));
116     PetscCall(DMPlexSetIsoperiodicFaceSF(dm, num_face_sfs, (PetscSF *)sfs)); // for the compose function effect on dm
117   }
118   PetscCall(DMDestroyLabelLinkList_Internal(dm));
119   PetscCall(DMCopyLabels(dmNew, dm, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL));
120   PetscCall(DMGetCoarseDM(dmNew, &coarseDM));
121   PetscCall(DMSetCoarseDM(dm, coarseDM));
122   PetscCall(DMDestroy(ndm));
123   PetscFunctionReturn(PETSC_SUCCESS);
124 }
125 
126 /* Swap dm with the contents of dmNew
127    - Swap the DM_Plex structure
128    - Swap the coordinates
129    - Swap the point PetscSF
130 */
131 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
132 {
133   DM          coordDMA, coordDMB;
134   Vec         coordsA, coordsB;
135   PetscSF     sfA, sfB;
136   DMField     fieldTmp;
137   void       *tmp;
138   DMLabelLink listTmp;
139   DMLabel     depthTmp;
140   PetscInt    tmpI;
141 
142   PetscFunctionBegin;
143   if (dmA == dmB) PetscFunctionReturn(PETSC_SUCCESS);
144   PetscCall(DMGetPointSF(dmA, &sfA));
145   PetscCall(DMGetPointSF(dmB, &sfB));
146   PetscCall(PetscObjectReference((PetscObject)sfA));
147   PetscCall(DMSetPointSF(dmA, sfB));
148   PetscCall(DMSetPointSF(dmB, sfA));
149   PetscCall(PetscObjectDereference((PetscObject)sfA));
150 
151   PetscCall(DMGetCoordinateDM(dmA, &coordDMA));
152   PetscCall(DMGetCoordinateDM(dmB, &coordDMB));
153   PetscCall(PetscObjectReference((PetscObject)coordDMA));
154   PetscCall(DMSetCoordinateDM(dmA, coordDMB));
155   PetscCall(DMSetCoordinateDM(dmB, coordDMA));
156   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
157 
158   PetscCall(DMGetCoordinatesLocal(dmA, &coordsA));
159   PetscCall(DMGetCoordinatesLocal(dmB, &coordsB));
160   PetscCall(PetscObjectReference((PetscObject)coordsA));
161   PetscCall(DMSetCoordinatesLocal(dmA, coordsB));
162   PetscCall(DMSetCoordinatesLocal(dmB, coordsA));
163   PetscCall(PetscObjectDereference((PetscObject)coordsA));
164 
165   PetscCall(DMGetCellCoordinateDM(dmA, &coordDMA));
166   PetscCall(DMGetCellCoordinateDM(dmB, &coordDMB));
167   PetscCall(PetscObjectReference((PetscObject)coordDMA));
168   PetscCall(DMSetCellCoordinateDM(dmA, coordDMB));
169   PetscCall(DMSetCellCoordinateDM(dmB, coordDMA));
170   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
171 
172   PetscCall(DMGetCellCoordinatesLocal(dmA, &coordsA));
173   PetscCall(DMGetCellCoordinatesLocal(dmB, &coordsB));
174   PetscCall(PetscObjectReference((PetscObject)coordsA));
175   PetscCall(DMSetCellCoordinatesLocal(dmA, coordsB));
176   PetscCall(DMSetCellCoordinatesLocal(dmB, coordsA));
177   PetscCall(PetscObjectDereference((PetscObject)coordsA));
178 
179   fieldTmp                  = dmA->coordinates[0].field;
180   dmA->coordinates[0].field = dmB->coordinates[0].field;
181   dmB->coordinates[0].field = fieldTmp;
182   fieldTmp                  = dmA->coordinates[1].field;
183   dmA->coordinates[1].field = dmB->coordinates[1].field;
184   dmB->coordinates[1].field = fieldTmp;
185   tmp                       = dmA->data;
186   dmA->data                 = dmB->data;
187   dmB->data                 = tmp;
188   listTmp                   = dmA->labels;
189   dmA->labels               = dmB->labels;
190   dmB->labels               = listTmp;
191   depthTmp                  = dmA->depthLabel;
192   dmA->depthLabel           = dmB->depthLabel;
193   dmB->depthLabel           = depthTmp;
194   depthTmp                  = dmA->celltypeLabel;
195   dmA->celltypeLabel        = dmB->celltypeLabel;
196   dmB->celltypeLabel        = depthTmp;
197   tmpI                      = dmA->levelup;
198   dmA->levelup              = dmB->levelup;
199   dmB->levelup              = tmpI;
200   PetscFunctionReturn(PETSC_SUCCESS);
201 }
202 
203 PetscErrorCode DMPlexInterpolateInPlace_Internal(DM dm)
204 {
205   DM idm;
206 
207   PetscFunctionBegin;
208   PetscCall(DMPlexInterpolate(dm, &idm));
209   PetscCall(DMPlexCopyCoordinates(dm, idm));
210   PetscCall(DMPlexReplace_Internal(dm, &idm));
211   PetscFunctionReturn(PETSC_SUCCESS);
212 }
213 
214 /*@C
215   DMPlexCreateCoordinateSpace - Creates a finite element space for the coordinates
216 
217   Collective
218 
219   Input Parameters:
220 + dm        - The `DMPLEX`
221 . degree    - The degree of the finite element or `PETSC_DECIDE`
222 . project   - Flag to project current coordinates into the space
223 - coordFunc - An optional function to map new points from refinement to the surface
224 
225   Level: advanced
226 
227 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `PetscPointFunc`, `PetscFECreateLagrange()`, `DMGetCoordinateDM()`
228 @*/
229 PetscErrorCode DMPlexCreateCoordinateSpace(DM dm, PetscInt degree, PetscBool project, PetscPointFunc coordFunc)
230 {
231   DM_Plex *mesh = (DM_Plex *)dm->data;
232   PetscFE  fe   = NULL;
233   DM       cdm;
234   PetscInt dim, dE, qorder, height;
235 
236   PetscFunctionBegin;
237   PetscCall(DMGetDimension(dm, &dim));
238   PetscCall(DMGetCoordinateDim(dm, &dE));
239   qorder = degree;
240   PetscCall(DMGetCoordinateDM(dm, &cdm));
241   PetscObjectOptionsBegin((PetscObject)cdm);
242   PetscCall(PetscOptionsBoundedInt("-default_quadrature_order", "Quadrature order is one less than quadrature points per edge", "DMPlexCreateCoordinateSpace", qorder, &qorder, NULL, 0));
243   PetscOptionsEnd();
244   PetscCall(DMPlexGetVTKCellHeight(dm, &height));
245   if (degree >= 0) {
246     DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
247     PetscInt       cStart, cEnd, gct;
248 
249     PetscCall(DMPlexGetHeightStratum(dm, height, &cStart, &cEnd));
250     if (cEnd > cStart) PetscCall(DMPlexGetCellType(dm, cStart, &ct));
251     gct = (PetscInt)ct;
252     PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &gct, 1, MPIU_INT, MPI_MIN, PetscObjectComm((PetscObject)dm)));
253     ct = (DMPolytopeType)gct;
254     // Work around current bug in PetscDualSpaceSetUp_Lagrange()
255     //   Can be seen in plex_tutorials-ex10_1
256     if (ct != DM_POLYTOPE_SEG_PRISM_TENSOR && ct != DM_POLYTOPE_TRI_PRISM_TENSOR && ct != DM_POLYTOPE_QUAD_PRISM_TENSOR) PetscCall(PetscFECreateLagrangeByCell(PETSC_COMM_SELF, dim, dE, ct, degree, qorder, &fe));
257   }
258   PetscCall(DMSetCoordinateDisc(dm, fe, project));
259   PetscCall(PetscFEDestroy(&fe));
260   mesh->coordFunc = coordFunc;
261   PetscFunctionReturn(PETSC_SUCCESS);
262 }
263 
264 /*@
265   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
266 
267   Collective
268 
269   Input Parameters:
270 + comm            - The communicator for the `DM` object
271 . dim             - The spatial dimension
272 . simplex         - Flag for simplicial cells, otherwise they are tensor product cells
273 . interpolate     - Flag to create intermediate mesh pieces (edges, faces)
274 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
275 
276   Output Parameter:
277 . newdm - The `DM` object
278 
279   Level: beginner
280 
281 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetType()`, `DMCreate()`
282 @*/
283 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscReal refinementLimit, DM *newdm)
284 {
285   DM          dm;
286   PetscMPIInt rank;
287 
288   PetscFunctionBegin;
289   PetscCall(DMCreate(comm, &dm));
290   PetscCall(DMSetType(dm, DMPLEX));
291   PetscCall(DMSetDimension(dm, dim));
292   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
293   PetscCallMPI(MPI_Comm_rank(comm, &rank));
294   switch (dim) {
295   case 2:
296     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "triangular"));
297     else PetscCall(PetscObjectSetName((PetscObject)dm, "quadrilateral"));
298     break;
299   case 3:
300     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "tetrahedral"));
301     else PetscCall(PetscObjectSetName((PetscObject)dm, "hexahedral"));
302     break;
303   default:
304     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
305   }
306   if (rank) {
307     PetscInt numPoints[2] = {0, 0};
308     PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL));
309   } else {
310     switch (dim) {
311     case 2:
312       if (simplex) {
313         PetscInt    numPoints[2]        = {4, 2};
314         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
315         PetscInt    cones[6]            = {2, 3, 4, 5, 4, 3};
316         PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
317         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
318 
319         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
320       } else {
321         PetscInt    numPoints[2]        = {6, 2};
322         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
323         PetscInt    cones[8]            = {2, 3, 4, 5, 3, 6, 7, 4};
324         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
325         PetscScalar vertexCoords[12]    = {-1.0, -0.5, 0.0, -0.5, 0.0, 0.5, -1.0, 0.5, 1.0, -0.5, 1.0, 0.5};
326 
327         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
328       }
329       break;
330     case 3:
331       if (simplex) {
332         PetscInt    numPoints[2]        = {5, 2};
333         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
334         PetscInt    cones[8]            = {4, 3, 5, 2, 5, 3, 4, 6};
335         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
336         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0};
337 
338         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
339       } else {
340         PetscInt    numPoints[2]         = {12, 2};
341         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
342         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9, 5, 4, 10, 11, 7, 12, 13, 8};
343         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
344         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5, -1.0, 0.5, -0.5, 0.0, 0.5, -0.5, 0.0, -0.5, -0.5, -1.0, -0.5, 0.5, 0.0, -0.5, 0.5, 0.0, 0.5, 0.5, -1.0, 0.5, 0.5, 1.0, 0.5, -0.5, 1.0, -0.5, -0.5, 1.0, -0.5, 0.5, 1.0, 0.5, 0.5};
345 
346         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
347       }
348       break;
349     default:
350       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
351     }
352   }
353   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
354   *newdm = dm;
355   if (refinementLimit > 0.0) {
356     DM          rdm;
357     const char *name;
358 
359     PetscCall(DMPlexSetRefinementUniform(*newdm, PETSC_FALSE));
360     PetscCall(DMPlexSetRefinementLimit(*newdm, refinementLimit));
361     PetscCall(DMRefine(*newdm, comm, &rdm));
362     PetscCall(PetscObjectGetName((PetscObject)*newdm, &name));
363     PetscCall(PetscObjectSetName((PetscObject)rdm, name));
364     PetscCall(DMDestroy(newdm));
365     *newdm = rdm;
366   }
367   if (interpolate) {
368     DM idm;
369 
370     PetscCall(DMPlexInterpolate(*newdm, &idm));
371     PetscCall(DMDestroy(newdm));
372     *newdm = idm;
373   }
374   PetscFunctionReturn(PETSC_SUCCESS);
375 }
376 
377 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
378 {
379   const PetscInt numVertices    = 2;
380   PetscInt       markerRight    = 1;
381   PetscInt       markerLeft     = 1;
382   PetscBool      markerSeparate = PETSC_FALSE;
383   Vec            coordinates;
384   PetscSection   coordSection;
385   PetscScalar   *coords;
386   PetscInt       coordSize;
387   PetscMPIInt    rank;
388   PetscInt       cdim = 1, v;
389 
390   PetscFunctionBegin;
391   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
392   if (markerSeparate) {
393     markerRight = 2;
394     markerLeft  = 1;
395   }
396   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
397   if (rank == 0) {
398     PetscCall(DMPlexSetChart(dm, 0, numVertices));
399     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
400     PetscCall(DMSetLabelValue(dm, "marker", 0, markerLeft));
401     PetscCall(DMSetLabelValue(dm, "marker", 1, markerRight));
402   }
403   PetscCall(DMPlexSymmetrize(dm));
404   PetscCall(DMPlexStratify(dm));
405   /* Build coordinates */
406   PetscCall(DMSetCoordinateDim(dm, cdim));
407   PetscCall(DMGetCoordinateSection(dm, &coordSection));
408   PetscCall(PetscSectionSetNumFields(coordSection, 1));
409   PetscCall(PetscSectionSetChart(coordSection, 0, numVertices));
410   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
411   for (v = 0; v < numVertices; ++v) {
412     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
413     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
414   }
415   PetscCall(PetscSectionSetUp(coordSection));
416   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
417   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
418   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
419   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
420   PetscCall(VecSetBlockSize(coordinates, cdim));
421   PetscCall(VecSetType(coordinates, VECSTANDARD));
422   PetscCall(VecGetArray(coordinates, &coords));
423   coords[0] = lower[0];
424   coords[1] = upper[0];
425   PetscCall(VecRestoreArray(coordinates, &coords));
426   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
427   PetscCall(VecDestroy(&coordinates));
428   PetscFunctionReturn(PETSC_SUCCESS);
429 }
430 
431 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
432 {
433   const PetscInt numVertices    = (edges[0] + 1) * (edges[1] + 1);
434   const PetscInt numEdges       = edges[0] * (edges[1] + 1) + (edges[0] + 1) * edges[1];
435   PetscInt       markerTop      = 1;
436   PetscInt       markerBottom   = 1;
437   PetscInt       markerRight    = 1;
438   PetscInt       markerLeft     = 1;
439   PetscBool      markerSeparate = PETSC_FALSE;
440   Vec            coordinates;
441   PetscSection   coordSection;
442   PetscScalar   *coords;
443   PetscInt       coordSize;
444   PetscMPIInt    rank;
445   PetscInt       v, vx, vy;
446 
447   PetscFunctionBegin;
448   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
449   if (markerSeparate) {
450     markerTop    = 3;
451     markerBottom = 1;
452     markerRight  = 2;
453     markerLeft   = 4;
454   }
455   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
456   if (rank == 0) {
457     PetscInt e, ex, ey;
458 
459     PetscCall(DMPlexSetChart(dm, 0, numEdges + numVertices));
460     for (e = 0; e < numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
461     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
462     for (vx = 0; vx <= edges[0]; vx++) {
463       for (ey = 0; ey < edges[1]; ey++) {
464         PetscInt edge   = vx * edges[1] + ey + edges[0] * (edges[1] + 1);
465         PetscInt vertex = ey * (edges[0] + 1) + vx + numEdges;
466         PetscInt cone[2];
467 
468         cone[0] = vertex;
469         cone[1] = vertex + edges[0] + 1;
470         PetscCall(DMPlexSetCone(dm, edge, cone));
471         if (vx == edges[0]) {
472           PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
473           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
474           if (ey == edges[1] - 1) {
475             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
476             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerRight));
477           }
478         } else if (vx == 0) {
479           PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
480           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
481           if (ey == edges[1] - 1) {
482             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
483             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft));
484           }
485         }
486       }
487     }
488     for (vy = 0; vy <= edges[1]; vy++) {
489       for (ex = 0; ex < edges[0]; ex++) {
490         PetscInt edge   = vy * edges[0] + ex;
491         PetscInt vertex = vy * (edges[0] + 1) + ex + numEdges;
492         PetscInt cone[2];
493 
494         cone[0] = vertex;
495         cone[1] = vertex + 1;
496         PetscCall(DMPlexSetCone(dm, edge, cone));
497         if (vy == edges[1]) {
498           PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
499           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
500           if (ex == edges[0] - 1) {
501             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
502             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerTop));
503           }
504         } else if (vy == 0) {
505           PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
506           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
507           if (ex == edges[0] - 1) {
508             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
509             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom));
510           }
511         }
512       }
513     }
514   }
515   PetscCall(DMPlexSymmetrize(dm));
516   PetscCall(DMPlexStratify(dm));
517   /* Build coordinates */
518   PetscCall(DMSetCoordinateDim(dm, 2));
519   PetscCall(DMGetCoordinateSection(dm, &coordSection));
520   PetscCall(PetscSectionSetNumFields(coordSection, 1));
521   PetscCall(PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices));
522   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 2));
523   for (v = numEdges; v < numEdges + numVertices; ++v) {
524     PetscCall(PetscSectionSetDof(coordSection, v, 2));
525     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 2));
526   }
527   PetscCall(PetscSectionSetUp(coordSection));
528   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
529   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
530   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
531   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
532   PetscCall(VecSetBlockSize(coordinates, 2));
533   PetscCall(VecSetType(coordinates, VECSTANDARD));
534   PetscCall(VecGetArray(coordinates, &coords));
535   for (vy = 0; vy <= edges[1]; ++vy) {
536     for (vx = 0; vx <= edges[0]; ++vx) {
537       coords[(vy * (edges[0] + 1) + vx) * 2 + 0] = lower[0] + ((upper[0] - lower[0]) / edges[0]) * vx;
538       coords[(vy * (edges[0] + 1) + vx) * 2 + 1] = lower[1] + ((upper[1] - lower[1]) / edges[1]) * vy;
539     }
540   }
541   PetscCall(VecRestoreArray(coordinates, &coords));
542   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
543   PetscCall(VecDestroy(&coordinates));
544   PetscFunctionReturn(PETSC_SUCCESS);
545 }
546 
547 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
548 {
549   PetscInt     vertices[3], numVertices;
550   PetscInt     numFaces       = 2 * faces[0] * faces[1] + 2 * faces[1] * faces[2] + 2 * faces[0] * faces[2];
551   PetscInt     markerTop      = 1;
552   PetscInt     markerBottom   = 1;
553   PetscInt     markerFront    = 1;
554   PetscInt     markerBack     = 1;
555   PetscInt     markerRight    = 1;
556   PetscInt     markerLeft     = 1;
557   PetscBool    markerSeparate = PETSC_FALSE;
558   Vec          coordinates;
559   PetscSection coordSection;
560   PetscScalar *coords;
561   PetscInt     coordSize;
562   PetscMPIInt  rank;
563   PetscInt     v, vx, vy, vz;
564   PetscInt     voffset, iface = 0, cone[4];
565 
566   PetscFunctionBegin;
567   PetscCheck(faces[0] >= 1 && faces[1] >= 1 && faces[2] >= 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
568   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
569   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
570   if (markerSeparate) {
571     markerBottom = 1;
572     markerTop    = 2;
573     markerFront  = 3;
574     markerBack   = 4;
575     markerRight  = 5;
576     markerLeft   = 6;
577   }
578   vertices[0] = faces[0] + 1;
579   vertices[1] = faces[1] + 1;
580   vertices[2] = faces[2] + 1;
581   numVertices = vertices[0] * vertices[1] * vertices[2];
582   if (rank == 0) {
583     PetscInt f;
584 
585     PetscCall(DMPlexSetChart(dm, 0, numFaces + numVertices));
586     for (f = 0; f < numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
587     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
588 
589     /* Side 0 (Top) */
590     for (vy = 0; vy < faces[1]; vy++) {
591       for (vx = 0; vx < faces[0]; vx++) {
592         voffset = numFaces + vertices[0] * vertices[1] * (vertices[2] - 1) + vy * vertices[0] + vx;
593         cone[0] = voffset;
594         cone[1] = voffset + 1;
595         cone[2] = voffset + vertices[0] + 1;
596         cone[3] = voffset + vertices[0];
597         PetscCall(DMPlexSetCone(dm, iface, cone));
598         PetscCall(DMSetLabelValue(dm, "marker", iface, markerTop));
599         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerTop));
600         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerTop));
601         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerTop));
602         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerTop));
603         iface++;
604       }
605     }
606 
607     /* Side 1 (Bottom) */
608     for (vy = 0; vy < faces[1]; vy++) {
609       for (vx = 0; vx < faces[0]; vx++) {
610         voffset = numFaces + vy * (faces[0] + 1) + vx;
611         cone[0] = voffset + 1;
612         cone[1] = voffset;
613         cone[2] = voffset + vertices[0];
614         cone[3] = voffset + vertices[0] + 1;
615         PetscCall(DMPlexSetCone(dm, iface, cone));
616         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBottom));
617         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBottom));
618         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBottom));
619         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerBottom));
620         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerBottom));
621         iface++;
622       }
623     }
624 
625     /* Side 2 (Front) */
626     for (vz = 0; vz < faces[2]; vz++) {
627       for (vx = 0; vx < faces[0]; vx++) {
628         voffset = numFaces + vz * vertices[0] * vertices[1] + vx;
629         cone[0] = voffset;
630         cone[1] = voffset + 1;
631         cone[2] = voffset + vertices[0] * vertices[1] + 1;
632         cone[3] = voffset + vertices[0] * vertices[1];
633         PetscCall(DMPlexSetCone(dm, iface, cone));
634         PetscCall(DMSetLabelValue(dm, "marker", iface, markerFront));
635         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerFront));
636         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerFront));
637         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerFront));
638         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerFront));
639         iface++;
640       }
641     }
642 
643     /* Side 3 (Back) */
644     for (vz = 0; vz < faces[2]; vz++) {
645       for (vx = 0; vx < faces[0]; vx++) {
646         voffset = numFaces + vz * vertices[0] * vertices[1] + vertices[0] * (vertices[1] - 1) + vx;
647         cone[0] = voffset + vertices[0] * vertices[1];
648         cone[1] = voffset + vertices[0] * vertices[1] + 1;
649         cone[2] = voffset + 1;
650         cone[3] = voffset;
651         PetscCall(DMPlexSetCone(dm, iface, cone));
652         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBack));
653         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBack));
654         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBack));
655         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerBack));
656         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerBack));
657         iface++;
658       }
659     }
660 
661     /* Side 4 (Left) */
662     for (vz = 0; vz < faces[2]; vz++) {
663       for (vy = 0; vy < faces[1]; vy++) {
664         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0];
665         cone[0] = voffset;
666         cone[1] = voffset + vertices[0] * vertices[1];
667         cone[2] = voffset + vertices[0] * vertices[1] + vertices[0];
668         cone[3] = voffset + vertices[0];
669         PetscCall(DMPlexSetCone(dm, iface, cone));
670         PetscCall(DMSetLabelValue(dm, "marker", iface, markerLeft));
671         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerLeft));
672         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerLeft));
673         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[1] + 0, markerLeft));
674         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerLeft));
675         iface++;
676       }
677     }
678 
679     /* Side 5 (Right) */
680     for (vz = 0; vz < faces[2]; vz++) {
681       for (vy = 0; vy < faces[1]; vy++) {
682         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0] + faces[0];
683         cone[0] = voffset + vertices[0] * vertices[1];
684         cone[1] = voffset;
685         cone[2] = voffset + vertices[0];
686         cone[3] = voffset + vertices[0] * vertices[1] + vertices[0];
687         PetscCall(DMPlexSetCone(dm, iface, cone));
688         PetscCall(DMSetLabelValue(dm, "marker", iface, markerRight));
689         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerRight));
690         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerRight));
691         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerRight));
692         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerRight));
693         iface++;
694       }
695     }
696   }
697   PetscCall(DMPlexSymmetrize(dm));
698   PetscCall(DMPlexStratify(dm));
699   /* Build coordinates */
700   PetscCall(DMSetCoordinateDim(dm, 3));
701   PetscCall(DMGetCoordinateSection(dm, &coordSection));
702   PetscCall(PetscSectionSetNumFields(coordSection, 1));
703   PetscCall(PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices));
704   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 3));
705   for (v = numFaces; v < numFaces + numVertices; ++v) {
706     PetscCall(PetscSectionSetDof(coordSection, v, 3));
707     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 3));
708   }
709   PetscCall(PetscSectionSetUp(coordSection));
710   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
711   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
712   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
713   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
714   PetscCall(VecSetBlockSize(coordinates, 3));
715   PetscCall(VecSetType(coordinates, VECSTANDARD));
716   PetscCall(VecGetArray(coordinates, &coords));
717   for (vz = 0; vz <= faces[2]; ++vz) {
718     for (vy = 0; vy <= faces[1]; ++vy) {
719       for (vx = 0; vx <= faces[0]; ++vx) {
720         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 0] = lower[0] + ((upper[0] - lower[0]) / faces[0]) * vx;
721         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 1] = lower[1] + ((upper[1] - lower[1]) / faces[1]) * vy;
722         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 2] = lower[2] + ((upper[2] - lower[2]) / faces[2]) * vz;
723       }
724     }
725   }
726   PetscCall(VecRestoreArray(coordinates, &coords));
727   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
728   PetscCall(VecDestroy(&coordinates));
729   PetscFunctionReturn(PETSC_SUCCESS);
730 }
731 
732 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate)
733 {
734   PetscFunctionBegin;
735   PetscValidLogicalCollectiveInt(dm, dim, 2);
736   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
737   PetscCall(DMSetDimension(dm, dim - 1));
738   PetscCall(DMSetCoordinateDim(dm, dim));
739   switch (dim) {
740   case 1:
741     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(dm, lower, upper, faces));
742     break;
743   case 2:
744     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(dm, lower, upper, faces));
745     break;
746   case 3:
747     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(dm, lower, upper, faces));
748     break;
749   default:
750     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Dimension not supported: %" PetscInt_FMT, dim);
751   }
752   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
753   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
754   PetscFunctionReturn(PETSC_SUCCESS);
755 }
756 
757 /*@C
758   DMPlexCreateBoxSurfaceMesh - Creates a mesh on the surface of the tensor product of unit intervals (box) using tensor cells (hexahedra).
759 
760   Collective
761 
762   Input Parameters:
763 + comm        - The communicator for the `DM` object
764 . dim         - The spatial dimension of the box, so the resulting mesh is has dimension `dim`-1
765 . faces       - Number of faces per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
766 . lower       - The lower left corner, or `NULL` for (0, 0, 0)
767 . upper       - The upper right corner, or `NULL` for (1, 1, 1)
768 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
769 
770   Output Parameter:
771 . dm - The `DM` object
772 
773   Level: beginner
774 
775 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateBoxMesh()`, `DMPlexCreateFromFile()`, `DMSetType()`, `DMCreate()`
776 @*/
777 PetscErrorCode DMPlexCreateBoxSurfaceMesh(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate, DM *dm)
778 {
779   PetscInt  fac[3] = {1, 1, 1};
780   PetscReal low[3] = {0, 0, 0};
781   PetscReal upp[3] = {1, 1, 1};
782 
783   PetscFunctionBegin;
784   PetscCall(DMCreate(comm, dm));
785   PetscCall(DMSetType(*dm, DMPLEX));
786   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(*dm, dim, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, interpolate));
787   PetscFunctionReturn(PETSC_SUCCESS);
788 }
789 
790 static PetscErrorCode DMPlexCreateLineMesh_Internal(DM dm, PetscInt segments, PetscReal lower, PetscReal upper, DMBoundaryType bd)
791 {
792   PetscInt     i, fStart, fEnd, numCells = 0, numVerts = 0;
793   PetscInt     numPoints[2], *coneSize, *cones, *coneOrientations;
794   PetscScalar *vertexCoords;
795   PetscReal    L, maxCell;
796   PetscBool    markerSeparate = PETSC_FALSE;
797   PetscInt     markerLeft = 1, faceMarkerLeft = 1;
798   PetscInt     markerRight = 1, faceMarkerRight = 2;
799   PetscBool    wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
800   PetscMPIInt  rank;
801 
802   PetscFunctionBegin;
803   PetscAssertPointer(dm, 1);
804 
805   PetscCall(DMSetDimension(dm, 1));
806   PetscCall(DMCreateLabel(dm, "marker"));
807   PetscCall(DMCreateLabel(dm, "Face Sets"));
808 
809   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
810   if (rank == 0) numCells = segments;
811   if (rank == 0) numVerts = segments + (wrap ? 0 : 1);
812 
813   numPoints[0] = numVerts;
814   numPoints[1] = numCells;
815   PetscCall(PetscMalloc4(numCells + numVerts, &coneSize, numCells * 2, &cones, numCells + numVerts, &coneOrientations, numVerts, &vertexCoords));
816   PetscCall(PetscArrayzero(coneOrientations, numCells + numVerts));
817   for (i = 0; i < numCells; ++i) coneSize[i] = 2;
818   for (i = 0; i < numVerts; ++i) coneSize[numCells + i] = 0;
819   for (i = 0; i < numCells; ++i) {
820     cones[2 * i]     = numCells + i % numVerts;
821     cones[2 * i + 1] = numCells + (i + 1) % numVerts;
822   }
823   for (i = 0; i < numVerts; ++i) vertexCoords[i] = lower + (upper - lower) * ((PetscReal)i / (PetscReal)numCells);
824   PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
825   PetscCall(PetscFree4(coneSize, cones, coneOrientations, vertexCoords));
826 
827   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
828   if (markerSeparate) {
829     markerLeft  = faceMarkerLeft;
830     markerRight = faceMarkerRight;
831   }
832   if (!wrap && rank == 0) {
833     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
834     PetscCall(DMSetLabelValue(dm, "marker", fStart, markerLeft));
835     PetscCall(DMSetLabelValue(dm, "marker", fEnd - 1, markerRight));
836     PetscCall(DMSetLabelValue(dm, "Face Sets", fStart, faceMarkerLeft));
837     PetscCall(DMSetLabelValue(dm, "Face Sets", fEnd - 1, faceMarkerRight));
838   }
839   if (wrap) {
840     L       = upper - lower;
841     maxCell = (PetscReal)1.1 * (L / (PetscReal)PetscMax(1, segments));
842     PetscCall(DMSetPeriodicity(dm, &maxCell, &lower, &L));
843   }
844   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
845   PetscFunctionReturn(PETSC_SUCCESS);
846 }
847 
848 // Creates "Face Sets" label based on the standard box labeling conventions
849 static PetscErrorCode DMPlexSetBoxLabel_Internal(DM dm, const DMBoundaryType periodicity[])
850 {
851   DM              cdm;
852   PetscSection    csection;
853   Vec             coordinates;
854   DMLabel         label;
855   IS              faces_is;
856   PetscInt        dim, num_face = 0;
857   const PetscInt *faces;
858   PetscInt        faceMarkerBottom, faceMarkerTop, faceMarkerFront, faceMarkerBack, faceMarkerRight, faceMarkerLeft;
859 
860   PetscFunctionBeginUser;
861   PetscCall(DMGetDimension(dm, &dim));
862   PetscCheck((dim == 2) || (dim == 3), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "DMPlex box labeling only supports 2D and 3D meshes, received DM of dimension %" PetscInt_FMT, dim);
863   // Get Face Sets label
864   PetscCall(DMGetLabel(dm, "Face Sets", &label));
865   if (label) {
866     PetscCall(DMLabelReset(label));
867   } else {
868     PetscCall(DMCreateLabel(dm, "Face Sets"));
869     PetscCall(DMGetLabel(dm, "Face Sets", &label));
870   }
871   PetscCall(DMPlexMarkBoundaryFaces(dm, 1, label));
872   PetscCall(DMGetStratumIS(dm, "Face Sets", 1, &faces_is));
873 
874   switch (dim) {
875   case 2:
876     faceMarkerTop    = 3;
877     faceMarkerBottom = 1;
878     faceMarkerRight  = 2;
879     faceMarkerLeft   = 4;
880     break;
881   case 3:
882     faceMarkerBottom = 1;
883     faceMarkerTop    = 2;
884     faceMarkerFront  = 3;
885     faceMarkerBack   = 4;
886     faceMarkerRight  = 5;
887     faceMarkerLeft   = 6;
888     break;
889   default:
890     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
891   }
892 
893   if (faces_is) PetscCall(ISGetLocalSize(faces_is, &num_face));
894   if (faces_is) PetscCall(ISGetIndices(faces_is, &faces));
895   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
896   PetscCall(DMGetCoordinateDM(dm, &cdm));
897   PetscCall(DMGetLocalSection(cdm, &csection));
898   for (PetscInt f = 0; f < num_face; ++f) {
899     PetscScalar *coords = NULL;
900     PetscInt     face = faces[f], flip = 1, label_value = -1, coords_size;
901 
902     { // Determine if orientation of face is flipped
903       PetscInt        num_cells_support, num_faces, start = -1;
904       const PetscInt *orients, *cell_faces, *cells;
905 
906       PetscCall(DMPlexGetSupport(dm, face, &cells));
907       PetscCall(DMPlexGetSupportSize(dm, face, &num_cells_support));
908       PetscCheck(num_cells_support == 1, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Expected one cell in support of exterior face, but got %" PetscInt_FMT " cells", num_cells_support);
909       PetscCall(DMPlexGetCone(dm, cells[0], &cell_faces));
910       PetscCall(DMPlexGetConeSize(dm, cells[0], &num_faces));
911       for (PetscInt i = 0; i < num_faces; i++) {
912         if (cell_faces[i] == face) start = i;
913       }
914       PetscCheck(start >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_CORRUPT, "Could not find face %" PetscInt_FMT " in cone of its support", face);
915       PetscCall(DMPlexGetConeOrientation(dm, cells[0], &orients));
916       if (orients[start] < 0) flip = -1;
917     }
918 
919     // Cannot use DMPlexComputeCellGeometryFVM() for high-order geometry, so must calculate normal vectors manually
920     // Use the vertices (depth 0) of coordinate DM to calculate normal vector
921     PetscCall(DMPlexVecGetClosureAtDepth_Internal(cdm, csection, coordinates, face, 0, &coords_size, &coords));
922     switch (dim) {
923     case 2: {
924       PetscScalar vec[2];
925 
926       for (PetscInt d = 0; d < dim; ++d) vec[d] = flip * (PetscRealPart(coords[1 * dim + d]) - PetscRealPart(coords[0 * dim + d]));
927       PetscScalar normal[] = {vec[1], -vec[0]};
928       if (PetscAbsScalar(normal[0]) > PetscAbsScalar(normal[1])) {
929         label_value = PetscRealPart(normal[0]) > 0 ? faceMarkerRight : faceMarkerLeft;
930       } else {
931         label_value = PetscRealPart(normal[1]) > 0 ? faceMarkerTop : faceMarkerBottom;
932       }
933     } break;
934     case 3: {
935       PetscScalar vec1[3], vec2[3], normal[3];
936 
937       for (PetscInt d = 0; d < dim; ++d) {
938         vec1[d] = PetscRealPart(coords[1 * dim + d]) - PetscRealPart(coords[0 * dim + d]);
939         vec2[d] = PetscRealPart(coords[2 * dim + d]) - PetscRealPart(coords[1 * dim + d]);
940       }
941 
942       // Calculate normal vector via cross-product
943       normal[0] = flip * ((vec1[1] * vec2[2]) - (vec1[2] * vec2[1]));
944       normal[1] = flip * ((vec1[2] * vec2[0]) - (vec1[0] * vec2[2]));
945       normal[2] = flip * ((vec1[0] * vec2[1]) - (vec1[1] * vec2[0]));
946 
947       if (PetscAbsScalar(normal[0]) > PetscAbsScalar(normal[1])) {
948         if (PetscAbsScalar(normal[0]) > PetscAbsScalar(normal[2])) {
949           label_value = PetscRealPart(normal[0]) > 0 ? faceMarkerRight : faceMarkerLeft;
950         } else {
951           label_value = PetscRealPart(normal[2]) > 0 ? faceMarkerTop : faceMarkerBottom;
952         }
953       } else {
954         if (PetscAbsScalar(normal[1]) > PetscAbsScalar(normal[2])) {
955           label_value = PetscRealPart(normal[1]) > 0 ? faceMarkerBack : faceMarkerFront;
956         } else {
957           label_value = PetscRealPart(normal[2]) > 0 ? faceMarkerTop : faceMarkerBottom;
958         }
959       }
960     } break;
961     }
962 
963     PetscInt previous_label_value; // always 1 due to DMPlexMarkBoundaryFaces call above
964     PetscCall(DMGetLabelValue(dm, "Face Sets", face, &previous_label_value));
965     PetscCall(DMClearLabelValue(dm, "Face Sets", face, previous_label_value));
966     PetscCall(DMSetLabelValue(dm, "Face Sets", face, label_value));
967     PetscCall(DMPlexVecRestoreClosure(cdm, csection, coordinates, face, &coords_size, &coords));
968   }
969   if (faces_is) PetscCall(ISRestoreIndices(faces_is, &faces));
970   PetscCall(ISDestroy(&faces_is));
971 
972   // Create Isoperiodic SF from newly-created face labels
973   PetscSF     periodicsfs[3];
974   PetscInt    periodic_sf_index  = 0;
975   PetscScalar transform[3][4][4] = {{{0.}}};
976   for (PetscInt d = 0; d < dim; d++) {
977     IS              donor_is, periodic_is;
978     const PetscInt *donor_faces = NULL, *periodic_faces = NULL;
979     PetscInt        num_donor = 0, num_periodic = 0;
980     PetscSF         centroidsf;
981     PetscReal       donor_to_periodic_distance;
982     const PetscInt  face_pairings[2][3][2] = {
983       // 2D face pairings, {donor, periodic}
984       {{4, 2}, {1, 3}},
985       // 3D face pairings
986       {{5, 6}, {3, 4}, {1, 2}}
987     };
988 
989     if (periodicity[d] != DM_BOUNDARY_PERIODIC) continue;
990     {
991       // Compute centroidsf, which is the mapping from donor faces to periodic faces
992       // Matches the centroid of the faces together, ignoring the periodic direction component (which should not match between donor and periodic face)
993       PetscInt     coords_size, centroid_comps = dim - 1;
994       PetscScalar *coords = NULL;
995       PetscReal   *donor_centroids, *periodic_centroids;
996       PetscReal    loc_periodic[2] = {PETSC_MIN_REAL, PETSC_MIN_REAL}, loc_periodic_global[2]; // Location of donor (0) and periodic (1) faces in periodic direction
997 
998       PetscCall(DMGetStratumIS(dm, "Face Sets", face_pairings[dim - 2][d][0], &donor_is));
999       PetscCall(DMGetStratumIS(dm, "Face Sets", face_pairings[dim - 2][d][1], &periodic_is));
1000       if (donor_is) {
1001         PetscCall(ISGetLocalSize(donor_is, &num_donor));
1002         PetscCall(ISGetIndices(donor_is, &donor_faces));
1003       }
1004       if (periodic_is) {
1005         PetscCall(ISGetLocalSize(periodic_is, &num_periodic));
1006         PetscCall(ISGetIndices(periodic_is, &periodic_faces));
1007       }
1008       PetscCall(PetscCalloc2(num_donor * centroid_comps, &donor_centroids, num_periodic * centroid_comps, &periodic_centroids));
1009       for (PetscInt f = 0; f < num_donor; f++) {
1010         PetscInt face = donor_faces[f], num_coords;
1011         PetscCall(DMPlexVecGetClosureAtDepth_Internal(cdm, csection, coordinates, face, 0, &coords_size, &coords));
1012         num_coords = coords_size / dim;
1013         for (PetscInt c = 0; c < num_coords; c++) {
1014           PetscInt comp_index = 0;
1015           loc_periodic[0]     = PetscRealPart(coords[c * dim + d]);
1016           for (PetscInt i = 0; i < dim; i++) {
1017             if (i == d) continue; // Periodic direction not used for centroid calculation
1018             donor_centroids[f * centroid_comps + comp_index] += PetscRealPart(coords[c * dim + i]) / num_coords;
1019             comp_index++;
1020           }
1021         }
1022         PetscCall(DMPlexVecRestoreClosure(cdm, csection, coordinates, face, &coords_size, &coords));
1023       }
1024 
1025       for (PetscInt f = 0; f < num_periodic; f++) {
1026         PetscInt face = periodic_faces[f], num_coords;
1027         PetscCall(DMPlexVecGetClosureAtDepth_Internal(cdm, csection, coordinates, face, 0, &coords_size, &coords));
1028         num_coords = coords_size / dim;
1029         for (PetscInt c = 0; c < num_coords; c++) {
1030           PetscInt comp_index = 0;
1031           loc_periodic[1]     = PetscRealPart(coords[c * dim + d]);
1032           for (PetscInt i = 0; i < dim; i++) {
1033             if (i == d) continue; // Periodic direction not used for centroid calculation
1034             periodic_centroids[f * centroid_comps + comp_index] += PetscRealPart(coords[c * dim + i]) / num_coords;
1035             comp_index++;
1036           }
1037         }
1038         PetscCall(DMPlexVecRestoreClosure(cdm, csection, coordinates, face, &coords_size, &coords));
1039       }
1040       PetscCallMPI(MPIU_Allreduce(loc_periodic, loc_periodic_global, 2, MPIU_REAL, MPIU_MAX, PetscObjectComm((PetscObject)dm)));
1041       donor_to_periodic_distance = loc_periodic_global[1] - loc_periodic_global[0];
1042 
1043       PetscCall(PetscSFCreate(PetscObjectComm((PetscObject)dm), &centroidsf));
1044       PetscCall(PetscSFSetGraphFromCoordinates(centroidsf, num_donor, num_periodic, centroid_comps, 1e-10, donor_centroids, periodic_centroids));
1045       PetscCall(PetscSFViewFromOptions(centroidsf, NULL, "-dm_plex_box_label_centroid_sf_view"));
1046       PetscCall(PetscFree2(donor_centroids, periodic_centroids));
1047     }
1048 
1049     { // Create Isoperiodic SF using centroidsSF
1050       PetscInt           pStart, pEnd;
1051       PetscInt          *leaf_faces;
1052       const PetscSFNode *firemote;
1053       PetscSFNode       *isoperiodic_leaves;
1054 
1055       PetscCall(PetscMalloc1(num_periodic, &leaf_faces));
1056       PetscCall(PetscSFBcastBegin(centroidsf, MPIU_INT, donor_faces, leaf_faces, MPI_REPLACE));
1057       PetscCall(PetscSFBcastEnd(centroidsf, MPIU_INT, donor_faces, leaf_faces, MPI_REPLACE));
1058 
1059       PetscCall(PetscMalloc1(num_periodic, &isoperiodic_leaves));
1060       PetscCall(PetscSFGetGraph(centroidsf, NULL, NULL, NULL, &firemote));
1061       for (PetscInt l = 0; l < num_periodic; ++l) {
1062         isoperiodic_leaves[l].index = leaf_faces[l];
1063         isoperiodic_leaves[l].rank  = firemote[l].rank;
1064       }
1065 
1066       PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
1067       PetscCall(PetscSFCreate(PetscObjectComm((PetscObject)dm), &periodicsfs[periodic_sf_index]));
1068       PetscCall(PetscSFSetGraph(periodicsfs[periodic_sf_index], pEnd - pStart, num_periodic, (PetscInt *)periodic_faces, PETSC_COPY_VALUES, isoperiodic_leaves, PETSC_OWN_POINTER));
1069       PetscCall(PetscSFViewFromOptions(periodicsfs[periodic_sf_index], NULL, "-dm_plex_box_label_periodic_sf_view"));
1070       PetscCall(PetscFree(leaf_faces));
1071     }
1072 
1073     transform[periodic_sf_index][0][0] = 1;
1074     transform[periodic_sf_index][1][1] = 1;
1075     transform[periodic_sf_index][2][2] = 1;
1076     transform[periodic_sf_index][3][3] = 1;
1077     transform[periodic_sf_index][d][3] = donor_to_periodic_distance;
1078 
1079     periodic_sf_index++;
1080     PetscCall(PetscSFDestroy(&centroidsf));
1081     if (donor_is) {
1082       PetscCall(ISRestoreIndices(donor_is, &donor_faces));
1083       PetscCall(ISDestroy(&donor_is));
1084     }
1085     if (periodic_is) {
1086       PetscCall(ISRestoreIndices(periodic_is, &periodic_faces));
1087       PetscCall(ISDestroy(&periodic_is));
1088     }
1089     PetscCall(DMClearLabelStratum(dm, "Face Sets", face_pairings[dim - 2][d][0]));
1090     PetscCall(DMClearLabelStratum(dm, "Face Sets", face_pairings[dim - 2][d][1]));
1091   }
1092   PetscCall(DMPlexSetIsoperiodicFaceSF(dm, periodic_sf_index, periodicsfs));
1093   PetscCall(DMPlexSetIsoperiodicFaceTransform(dm, periodic_sf_index, (const PetscScalar *)transform));
1094   for (PetscInt p = 0; p < periodic_sf_index; p++) PetscCall(PetscSFDestroy(&periodicsfs[p]));
1095 
1096   { // Update coordinate DM with new Face Sets label
1097     DM      cdm;
1098     DMLabel oldFaceSets, newFaceSets;
1099     PetscCall(DMGetCoordinateDM(dm, &cdm));
1100     PetscCall(DMGetLabel(cdm, "Face Sets", &oldFaceSets));
1101     if (oldFaceSets) PetscCall(DMRemoveLabelBySelf(cdm, &oldFaceSets, PETSC_FALSE));
1102     PetscCall(DMLabelDuplicate(label, &newFaceSets));
1103     PetscCall(DMAddLabel(cdm, newFaceSets));
1104     PetscCall(DMLabelDestroy(&newFaceSets));
1105   }
1106   PetscFunctionReturn(PETSC_SUCCESS);
1107 }
1108 
1109 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1110 {
1111   DM      boundary, vol;
1112   DMLabel bdlabel;
1113 
1114   PetscFunctionBegin;
1115   PetscAssertPointer(dm, 1);
1116   for (PetscInt i = 0; i < dim; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
1117   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &boundary));
1118   PetscCall(DMSetType(boundary, DMPLEX));
1119   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(boundary, dim, faces, lower, upper, PETSC_FALSE));
1120   PetscCall(DMPlexGenerate(boundary, NULL, interpolate, &vol));
1121   PetscCall(DMGetLabel(vol, "marker", &bdlabel));
1122   if (bdlabel) PetscCall(DMPlexLabelComplete(vol, bdlabel));
1123   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_FALSE, vol));
1124   PetscCall(DMPlexReplace_Internal(dm, &vol));
1125   if (interpolate) {
1126     PetscCall(DMPlexInterpolateInPlace_Internal(dm));
1127     PetscCall(DMPlexSetBoxLabel_Internal(dm, periodicity));
1128   }
1129   PetscCall(DMDestroy(&boundary));
1130   PetscFunctionReturn(PETSC_SUCCESS);
1131 }
1132 
1133 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
1134 {
1135   DMLabel     cutLabel  = NULL;
1136   PetscInt    markerTop = 1, faceMarkerTop = 1;
1137   PetscInt    markerBottom = 1, faceMarkerBottom = 1;
1138   PetscInt    markerFront = 1, faceMarkerFront = 1;
1139   PetscInt    markerBack = 1, faceMarkerBack = 1;
1140   PetscInt    markerRight = 1, faceMarkerRight = 1;
1141   PetscInt    markerLeft = 1, faceMarkerLeft = 1;
1142   PetscInt    dim;
1143   PetscBool   markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
1144   PetscMPIInt rank;
1145 
1146   PetscFunctionBegin;
1147   PetscCall(DMGetDimension(dm, &dim));
1148   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1149   PetscCall(DMCreateLabel(dm, "marker"));
1150   PetscCall(DMCreateLabel(dm, "Face Sets"));
1151   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
1152   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST || bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST || bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
1153     if (cutMarker) {
1154       PetscCall(DMCreateLabel(dm, "periodic_cut"));
1155       PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
1156     }
1157   }
1158   switch (dim) {
1159   case 2:
1160     faceMarkerTop    = 3;
1161     faceMarkerBottom = 1;
1162     faceMarkerRight  = 2;
1163     faceMarkerLeft   = 4;
1164     break;
1165   case 3:
1166     faceMarkerBottom = 1;
1167     faceMarkerTop    = 2;
1168     faceMarkerFront  = 3;
1169     faceMarkerBack   = 4;
1170     faceMarkerRight  = 5;
1171     faceMarkerLeft   = 6;
1172     break;
1173   default:
1174     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
1175   }
1176   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
1177   if (markerSeparate) {
1178     markerBottom = faceMarkerBottom;
1179     markerTop    = faceMarkerTop;
1180     markerFront  = faceMarkerFront;
1181     markerBack   = faceMarkerBack;
1182     markerRight  = faceMarkerRight;
1183     markerLeft   = faceMarkerLeft;
1184   }
1185   {
1186     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
1187     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
1188     const PetscInt numZEdges    = rank == 0 ? edges[2] : 0;
1189     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0] + 1) : 0;
1190     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1] + 1) : 0;
1191     const PetscInt numZVertices = rank == 0 ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2] + 1) : 0;
1192     const PetscInt numCells     = numXEdges * numYEdges * numZEdges;
1193     const PetscInt numXFaces    = numYEdges * numZEdges;
1194     const PetscInt numYFaces    = numXEdges * numZEdges;
1195     const PetscInt numZFaces    = numXEdges * numYEdges;
1196     const PetscInt numTotXFaces = numXVertices * numXFaces;
1197     const PetscInt numTotYFaces = numYVertices * numYFaces;
1198     const PetscInt numTotZFaces = numZVertices * numZFaces;
1199     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
1200     const PetscInt numTotXEdges = numXEdges * numYVertices * numZVertices;
1201     const PetscInt numTotYEdges = numYEdges * numXVertices * numZVertices;
1202     const PetscInt numTotZEdges = numZEdges * numXVertices * numYVertices;
1203     const PetscInt numVertices  = numXVertices * numYVertices * numZVertices;
1204     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
1205     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
1206     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
1207     const PetscInt firstYFace   = firstXFace + numTotXFaces;
1208     const PetscInt firstZFace   = firstYFace + numTotYFaces;
1209     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
1210     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
1211     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
1212     Vec            coordinates;
1213     PetscSection   coordSection;
1214     PetscScalar   *coords;
1215     PetscInt       coordSize;
1216     PetscInt       v, vx, vy, vz;
1217     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
1218 
1219     PetscCall(DMPlexSetChart(dm, 0, numCells + numFaces + numEdges + numVertices));
1220     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
1221     for (f = firstXFace; f < firstXFace + numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
1222     for (e = firstXEdge; e < firstXEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
1223     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
1224     /* Build cells */
1225     for (fz = 0; fz < numZEdges; ++fz) {
1226       for (fy = 0; fy < numYEdges; ++fy) {
1227         for (fx = 0; fx < numXEdges; ++fx) {
1228           PetscInt cell  = (fz * numYEdges + fy) * numXEdges + fx;
1229           PetscInt faceB = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
1230           PetscInt faceT = firstZFace + (fy * numXEdges + fx) * numZVertices + ((fz + 1) % numZVertices);
1231           PetscInt faceF = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
1232           PetscInt faceK = firstYFace + (fz * numXEdges + fx) * numYVertices + ((fy + 1) % numYVertices);
1233           PetscInt faceL = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
1234           PetscInt faceR = firstXFace + (fz * numYEdges + fy) * numXVertices + ((fx + 1) % numXVertices);
1235           /* B,  T,  F,  K,  R,  L */
1236           PetscInt ornt[6] = {-2, 0, 0, -3, 0, -2}; /* ??? */
1237           PetscInt cone[6];
1238 
1239           /* no boundary twisting in 3D */
1240           cone[0] = faceB;
1241           cone[1] = faceT;
1242           cone[2] = faceF;
1243           cone[3] = faceK;
1244           cone[4] = faceR;
1245           cone[5] = faceL;
1246           PetscCall(DMPlexSetCone(dm, cell, cone));
1247           PetscCall(DMPlexSetConeOrientation(dm, cell, ornt));
1248           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
1249           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
1250           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
1251         }
1252       }
1253     }
1254     /* Build x faces */
1255     for (fz = 0; fz < numZEdges; ++fz) {
1256       for (fy = 0; fy < numYEdges; ++fy) {
1257         for (fx = 0; fx < numXVertices; ++fx) {
1258           PetscInt face    = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
1259           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
1260           PetscInt edgeR   = firstZEdge + (((fy + 1) % numYVertices) * numXVertices + fx) * numZEdges + fz;
1261           PetscInt edgeB   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
1262           PetscInt edgeT   = firstYEdge + (((fz + 1) % numZVertices) * numXVertices + fx) * numYEdges + fy;
1263           PetscInt ornt[4] = {0, 0, -1, -1};
1264           PetscInt cone[4];
1265 
1266           if (dim == 3) {
1267             /* markers */
1268             if (bdX != DM_BOUNDARY_PERIODIC) {
1269               if (fx == numXVertices - 1) {
1270                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight));
1271                 PetscCall(DMSetLabelValue(dm, "marker", face, markerRight));
1272               } else if (fx == 0) {
1273                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft));
1274                 PetscCall(DMSetLabelValue(dm, "marker", face, markerLeft));
1275               }
1276             }
1277           }
1278           cone[0] = edgeB;
1279           cone[1] = edgeR;
1280           cone[2] = edgeT;
1281           cone[3] = edgeL;
1282           PetscCall(DMPlexSetCone(dm, face, cone));
1283           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1284         }
1285       }
1286     }
1287     /* Build y faces */
1288     for (fz = 0; fz < numZEdges; ++fz) {
1289       for (fx = 0; fx < numXEdges; ++fx) {
1290         for (fy = 0; fy < numYVertices; ++fy) {
1291           PetscInt face    = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
1292           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
1293           PetscInt edgeR   = firstZEdge + (fy * numXVertices + ((fx + 1) % numXVertices)) * numZEdges + fz;
1294           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1295           PetscInt edgeT   = firstXEdge + (((fz + 1) % numZVertices) * numYVertices + fy) * numXEdges + fx;
1296           PetscInt ornt[4] = {0, 0, -1, -1};
1297           PetscInt cone[4];
1298 
1299           if (dim == 3) {
1300             /* markers */
1301             if (bdY != DM_BOUNDARY_PERIODIC) {
1302               if (fy == numYVertices - 1) {
1303                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack));
1304                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBack));
1305               } else if (fy == 0) {
1306                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront));
1307                 PetscCall(DMSetLabelValue(dm, "marker", face, markerFront));
1308               }
1309             }
1310           }
1311           cone[0] = edgeB;
1312           cone[1] = edgeR;
1313           cone[2] = edgeT;
1314           cone[3] = edgeL;
1315           PetscCall(DMPlexSetCone(dm, face, cone));
1316           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1317         }
1318       }
1319     }
1320     /* Build z faces */
1321     for (fy = 0; fy < numYEdges; ++fy) {
1322       for (fx = 0; fx < numXEdges; ++fx) {
1323         for (fz = 0; fz < numZVertices; fz++) {
1324           PetscInt face    = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
1325           PetscInt edgeL   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
1326           PetscInt edgeR   = firstYEdge + (fz * numXVertices + ((fx + 1) % numXVertices)) * numYEdges + fy;
1327           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1328           PetscInt edgeT   = firstXEdge + (fz * numYVertices + ((fy + 1) % numYVertices)) * numXEdges + fx;
1329           PetscInt ornt[4] = {0, 0, -1, -1};
1330           PetscInt cone[4];
1331 
1332           if (dim == 2) {
1333             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges - 1) {
1334               edgeR += numYEdges - 1 - 2 * fy;
1335               ornt[1] = -1;
1336             }
1337             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges - 1) {
1338               edgeT += numXEdges - 1 - 2 * fx;
1339               ornt[2] = 0;
1340             }
1341             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1342             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1343           } else {
1344             /* markers */
1345             if (bdZ != DM_BOUNDARY_PERIODIC) {
1346               if (fz == numZVertices - 1) {
1347                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop));
1348                 PetscCall(DMSetLabelValue(dm, "marker", face, markerTop));
1349               } else if (fz == 0) {
1350                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom));
1351                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBottom));
1352               }
1353             }
1354           }
1355           cone[0] = edgeB;
1356           cone[1] = edgeR;
1357           cone[2] = edgeT;
1358           cone[3] = edgeL;
1359           PetscCall(DMPlexSetCone(dm, face, cone));
1360           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1361         }
1362       }
1363     }
1364     /* Build Z edges*/
1365     for (vy = 0; vy < numYVertices; vy++) {
1366       for (vx = 0; vx < numXVertices; vx++) {
1367         for (ez = 0; ez < numZEdges; ez++) {
1368           const PetscInt edge    = firstZEdge + (vy * numXVertices + vx) * numZEdges + ez;
1369           const PetscInt vertexB = firstVertex + (ez * numYVertices + vy) * numXVertices + vx;
1370           const PetscInt vertexT = firstVertex + (((ez + 1) % numZVertices) * numYVertices + vy) * numXVertices + vx;
1371           PetscInt       cone[2];
1372 
1373           cone[0] = vertexB;
1374           cone[1] = vertexT;
1375           PetscCall(DMPlexSetCone(dm, edge, cone));
1376           if (dim == 3) {
1377             if (bdX != DM_BOUNDARY_PERIODIC) {
1378               if (vx == numXVertices - 1) {
1379                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1380                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1381                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1382               } else if (vx == 0) {
1383                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1384                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1385                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1386               }
1387             }
1388             if (bdY != DM_BOUNDARY_PERIODIC) {
1389               if (vy == numYVertices - 1) {
1390                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1391                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1392                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1393               } else if (vy == 0) {
1394                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1395                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1396                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1397               }
1398             }
1399           }
1400         }
1401       }
1402     }
1403     /* Build Y edges*/
1404     for (vz = 0; vz < numZVertices; vz++) {
1405       for (vx = 0; vx < numXVertices; vx++) {
1406         for (ey = 0; ey < numYEdges; ey++) {
1407           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges - 1) ? (numXVertices - vx - 1) : (vz * numYVertices + ((ey + 1) % numYVertices)) * numXVertices + vx;
1408           const PetscInt edge    = firstYEdge + (vz * numXVertices + vx) * numYEdges + ey;
1409           const PetscInt vertexF = firstVertex + (vz * numYVertices + ey) * numXVertices + vx;
1410           const PetscInt vertexK = firstVertex + nextv;
1411           PetscInt       cone[2];
1412 
1413           cone[0] = vertexF;
1414           cone[1] = vertexK;
1415           PetscCall(DMPlexSetCone(dm, edge, cone));
1416           if (dim == 2) {
1417             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1418               if (vx == numXVertices - 1) {
1419                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight));
1420                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1421                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1422                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1423               } else if (vx == 0) {
1424                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft));
1425                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1426                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1427                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1428               }
1429             } else {
1430               if (vx == 0 && cutLabel) {
1431                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1432                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1433                 if (ey == numYEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1434               }
1435             }
1436           } else {
1437             if (bdX != DM_BOUNDARY_PERIODIC) {
1438               if (vx == numXVertices - 1) {
1439                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1440                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1441                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1442               } else if (vx == 0) {
1443                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1444                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1445                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1446               }
1447             }
1448             if (bdZ != DM_BOUNDARY_PERIODIC) {
1449               if (vz == numZVertices - 1) {
1450                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1451                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1452                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1453               } else if (vz == 0) {
1454                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1455                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1456                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1457               }
1458             }
1459           }
1460         }
1461       }
1462     }
1463     /* Build X edges*/
1464     for (vz = 0; vz < numZVertices; vz++) {
1465       for (vy = 0; vy < numYVertices; vy++) {
1466         for (ex = 0; ex < numXEdges; ex++) {
1467           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges - 1) ? (numYVertices - vy - 1) * numXVertices : (vz * numYVertices + vy) * numXVertices + (ex + 1) % numXVertices;
1468           const PetscInt edge    = firstXEdge + (vz * numYVertices + vy) * numXEdges + ex;
1469           const PetscInt vertexL = firstVertex + (vz * numYVertices + vy) * numXVertices + ex;
1470           const PetscInt vertexR = firstVertex + nextv;
1471           PetscInt       cone[2];
1472 
1473           cone[0] = vertexL;
1474           cone[1] = vertexR;
1475           PetscCall(DMPlexSetCone(dm, edge, cone));
1476           if (dim == 2) {
1477             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1478               if (vy == numYVertices - 1) {
1479                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop));
1480                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1481                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1482                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1483               } else if (vy == 0) {
1484                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom));
1485                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1486                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1487                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1488               }
1489             } else {
1490               if (vy == 0 && cutLabel) {
1491                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1492                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1493                 if (ex == numXEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1494               }
1495             }
1496           } else {
1497             if (bdY != DM_BOUNDARY_PERIODIC) {
1498               if (vy == numYVertices - 1) {
1499                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1500                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1501                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1502               } else if (vy == 0) {
1503                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1504                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1505                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1506               }
1507             }
1508             if (bdZ != DM_BOUNDARY_PERIODIC) {
1509               if (vz == numZVertices - 1) {
1510                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1511                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1512                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1513               } else if (vz == 0) {
1514                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1515                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1516                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1517               }
1518             }
1519           }
1520         }
1521       }
1522     }
1523     PetscCall(DMPlexSymmetrize(dm));
1524     PetscCall(DMPlexStratify(dm));
1525     /* Build coordinates */
1526     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1527     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1528     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1529     PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVertices));
1530     for (v = firstVertex; v < firstVertex + numVertices; ++v) {
1531       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1532       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1533     }
1534     PetscCall(PetscSectionSetUp(coordSection));
1535     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1536     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1537     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1538     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1539     PetscCall(VecSetBlockSize(coordinates, dim));
1540     PetscCall(VecSetType(coordinates, VECSTANDARD));
1541     PetscCall(VecGetArray(coordinates, &coords));
1542     for (vz = 0; vz < numZVertices; ++vz) {
1543       for (vy = 0; vy < numYVertices; ++vy) {
1544         for (vx = 0; vx < numXVertices; ++vx) {
1545           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * vx;
1546           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * vy;
1547           if (dim == 3) coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 2] = lower[2] + ((upper[2] - lower[2]) / numZEdges) * vz;
1548         }
1549       }
1550     }
1551     PetscCall(VecRestoreArray(coordinates, &coords));
1552     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1553     PetscCall(VecDestroy(&coordinates));
1554   }
1555   PetscFunctionReturn(PETSC_SUCCESS);
1556 }
1557 
1558 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1559 {
1560   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1561   PetscInt       fac[3] = {0, 0, 0}, d;
1562 
1563   PetscFunctionBegin;
1564   PetscAssertPointer(dm, 1);
1565   PetscValidLogicalCollectiveInt(dm, dim, 2);
1566   PetscCall(DMSetDimension(dm, dim));
1567   for (d = 0; d < dim; ++d) {
1568     fac[d] = faces[d];
1569     bdt[d] = periodicity[d];
1570   }
1571   PetscCall(DMPlexCreateCubeMesh_Internal(dm, lower, upper, fac, bdt[0], bdt[1], bdt[2]));
1572   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST || periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST || (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
1573     PetscReal L[3]       = {-1., -1., 0.};
1574     PetscReal maxCell[3] = {-1., -1., 0.};
1575 
1576     for (d = 0; d < dim; ++d) {
1577       if (periodicity[d] != DM_BOUNDARY_NONE) {
1578         L[d]       = upper[d] - lower[d];
1579         maxCell[d] = 1.1 * (L[d] / PetscMax(1, faces[d]));
1580       }
1581     }
1582     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1583   }
1584   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
1585   PetscFunctionReturn(PETSC_SUCCESS);
1586 }
1587 
1588 static PetscErrorCode DMPlexCreateBoxMesh_Internal(DM dm, DMPlexShape shape, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1589 {
1590   PetscFunctionBegin;
1591   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
1592   if (shape == DM_SHAPE_ZBOX) PetscCall(DMPlexCreateBoxMesh_Tensor_SFC_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1593   else if (dim == 1) PetscCall(DMPlexCreateLineMesh_Internal(dm, faces[0], lower[0], upper[0], periodicity[0]));
1594   else if (simplex) PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1595   else PetscCall(DMPlexCreateBoxMesh_Tensor_Internal(dm, dim, faces, lower, upper, periodicity));
1596   if (!interpolate && dim > 1 && !simplex) {
1597     DM udm;
1598 
1599     PetscCall(DMPlexUninterpolate(dm, &udm));
1600     PetscCall(DMPlexCopyCoordinates(dm, udm));
1601     PetscCall(DMPlexReplace_Internal(dm, &udm));
1602   }
1603   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
1604   PetscFunctionReturn(PETSC_SUCCESS);
1605 }
1606 
1607 /*@
1608   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
1609 
1610   Collective
1611 
1612   Input Parameters:
1613 + comm               - The communicator for the `DM` object
1614 . dim                - The spatial dimension
1615 . simplex            - `PETSC_TRUE` for simplices, `PETSC_FALSE` for tensor cells
1616 . faces              - Number of faces per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1617 . lower              - The lower left corner, or `NULL` for (0, 0, 0)
1618 . upper              - The upper right corner, or `NULL` for (1, 1, 1)
1619 . periodicity        - The boundary type for the X,Y,Z direction, or `NULL` for `DM_BOUNDARY_NONE`
1620 . interpolate        - Flag to create intermediate mesh pieces (edges, faces)
1621 . localizationHeight - Flag to localize edges and faces in addition to cells; only significant for periodic meshes
1622 - sparseLocalize     - Flag to localize coordinates only for cells near the periodic boundary; only significant for periodic meshes
1623 
1624   Output Parameter:
1625 . dm - The `DM` object
1626 
1627   Level: beginner
1628 
1629   Note:
1630   To customize this mesh using options, use
1631 .vb
1632   DMCreate(comm, &dm);
1633   DMSetType(dm, DMPLEX);
1634   DMSetFromOptions(dm);
1635 .ve
1636   and use the options in `DMSetFromOptions()`.
1637 
1638   Here is the numbering returned for 2 faces in each direction for tensor cells\:
1639 .vb
1640  10---17---11---18----12
1641   |         |         |
1642   |         |         |
1643  20    2   22    3    24
1644   |         |         |
1645   |         |         |
1646   7---15----8---16----9
1647   |         |         |
1648   |         |         |
1649  19    0   21    1   23
1650   |         |         |
1651   |         |         |
1652   4---13----5---14----6
1653 .ve
1654   and for simplicial cells
1655 .vb
1656  14----8---15----9----16
1657   |\     5  |\      7 |
1658   | \       | \       |
1659  13   2    14    3    15
1660   | 4   \   | 6   \   |
1661   |       \ |       \ |
1662  11----6---12----7----13
1663   |\        |\        |
1664   | \    1  | \     3 |
1665  10   0    11    1    12
1666   | 0   \   | 2   \   |
1667   |       \ |       \ |
1668   8----4----9----5----10
1669 .ve
1670 
1671 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
1672 @*/
1673 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, PetscInt localizationHeight, PetscBool sparseLocalize, DM *dm)
1674 {
1675   PetscInt       fac[3] = {1, 1, 1};
1676   PetscReal      low[3] = {0, 0, 0};
1677   PetscReal      upp[3] = {1, 1, 1};
1678   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1679 
1680   PetscFunctionBegin;
1681   PetscCall(DMCreate(comm, dm));
1682   PetscCall(DMSetType(*dm, DMPLEX));
1683   PetscCall(DMPlexCreateBoxMesh_Internal(*dm, DM_SHAPE_BOX, dim, simplex, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt, interpolate));
1684   if (periodicity) {
1685     DM cdm;
1686 
1687     PetscCall(DMGetCoordinateDM(*dm, &cdm));
1688     PetscCall(DMPlexSetMaxProjectionHeight(cdm, localizationHeight));
1689     PetscCall(DMSetSparseLocalize(*dm, sparseLocalize));
1690     PetscCall(DMLocalizeCoordinates(*dm));
1691   }
1692   PetscFunctionReturn(PETSC_SUCCESS);
1693 }
1694 
1695 static PetscErrorCode DMPlexCreateWedgeBoxMesh_Internal(DM dm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1696 {
1697   DM       bdm, vol;
1698   PetscInt i;
1699 
1700   PetscFunctionBegin;
1701   // TODO Now we can support periodicity
1702   for (i = 0; i < 3; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity not yet supported");
1703   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &bdm));
1704   PetscCall(DMSetType(bdm, DMPLEX));
1705   PetscCall(DMSetDimension(bdm, 2));
1706   PetscCall(PetscLogEventBegin(DMPLEX_Generate, bdm, 0, 0, 0));
1707   PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(bdm, 2, faces, lower, upper, periodicity, PETSC_TRUE));
1708   PetscCall(DMPlexExtrude(bdm, faces[2], upper[2] - lower[2], PETSC_TRUE, PETSC_FALSE, PETSC_FALSE, NULL, NULL, &vol));
1709   PetscCall(PetscLogEventEnd(DMPLEX_Generate, bdm, 0, 0, 0));
1710   PetscCall(DMDestroy(&bdm));
1711   PetscCall(DMPlexReplace_Internal(dm, &vol));
1712   if (lower[2] != 0.0) {
1713     Vec          v;
1714     PetscScalar *x;
1715     PetscInt     cDim, n;
1716 
1717     PetscCall(DMGetCoordinatesLocal(dm, &v));
1718     PetscCall(VecGetBlockSize(v, &cDim));
1719     PetscCall(VecGetLocalSize(v, &n));
1720     PetscCall(VecGetArray(v, &x));
1721     x += cDim;
1722     for (i = 0; i < n; i += cDim) x[i] += lower[2];
1723     PetscCall(VecRestoreArray(v, &x));
1724     PetscCall(DMSetCoordinatesLocal(dm, v));
1725   }
1726   PetscFunctionReturn(PETSC_SUCCESS);
1727 }
1728 
1729 /*@
1730   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tessellating the (x,y) plane and extruding in the third direction using wedge cells.
1731 
1732   Collective
1733 
1734   Input Parameters:
1735 + comm        - The communicator for the `DM` object
1736 . faces       - Number of faces per dimension, or `NULL` for (1, 1, 1)
1737 . lower       - The lower left corner, or `NULL` for (0, 0, 0)
1738 . upper       - The upper right corner, or `NULL` for (1, 1, 1)
1739 . periodicity - The boundary type for the X,Y,Z direction, or `NULL` for `DM_BOUNDARY_NONE`
1740 . orderHeight - If `PETSC_TRUE`, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1741 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1742 
1743   Output Parameter:
1744 . dm - The `DM` object
1745 
1746   Level: beginner
1747 
1748 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateWedgeCylinderMesh()`, `DMExtrude()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1749 @*/
1750 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool orderHeight, PetscBool interpolate, DM *dm)
1751 {
1752   PetscInt       fac[3] = {1, 1, 1};
1753   PetscReal      low[3] = {0, 0, 0};
1754   PetscReal      upp[3] = {1, 1, 1};
1755   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1756 
1757   PetscFunctionBegin;
1758   PetscCall(DMCreate(comm, dm));
1759   PetscCall(DMSetType(*dm, DMPLEX));
1760   PetscCall(DMPlexCreateWedgeBoxMesh_Internal(*dm, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt));
1761   if (!interpolate) {
1762     DM udm;
1763 
1764     PetscCall(DMPlexUninterpolate(*dm, &udm));
1765     PetscCall(DMPlexReplace_Internal(*dm, &udm));
1766   }
1767   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1768   PetscFunctionReturn(PETSC_SUCCESS);
1769 }
1770 
1771 /*
1772   DMPlexTensorPointLexicographic_Private - Returns all tuples of size 'len' with nonnegative integers that are all less than or equal to 'max' for that dimension.
1773 
1774   Input Parameters:
1775 + len - The length of the tuple
1776 . max - The maximum for each dimension, so values are in [0, max)
1777 - tup - A tuple of length len+1: tup[len] > 0 indicates a stopping condition
1778 
1779   Output Parameter:
1780 . tup - A tuple of `len` integers whose entries are at most `max`
1781 
1782   Level: developer
1783 
1784   Note:
1785   Ordering is lexicographic with lowest index as least significant in ordering.
1786   e.g. for len == 2 and max == 2, this will return, in order, {0,0}, {1,0}, {2,0}, {0,1}, {1,1}, {2,1}, {0,2}, {1,2}, {2,2}.
1787 
1788 .seealso: PetscDualSpaceTensorPointLexicographic_Internal(), PetscDualSpaceLatticePointLexicographic_Internal()
1789 */
1790 static PetscErrorCode DMPlexTensorPointLexicographic_Private(PetscInt len, const PetscInt max[], PetscInt tup[])
1791 {
1792   PetscInt i;
1793 
1794   PetscFunctionBegin;
1795   for (i = 0; i < len; ++i) {
1796     if (tup[i] < max[i] - 1) {
1797       break;
1798     } else {
1799       tup[i] = 0;
1800     }
1801   }
1802   if (i == len) tup[i - 1] = max[i - 1];
1803   else ++tup[i];
1804   PetscFunctionReturn(PETSC_SUCCESS);
1805 }
1806 
1807 static PetscInt TupleToIndex_Private(PetscInt len, const PetscInt max[], const PetscInt tup[])
1808 {
1809   PetscInt i, idx = tup[len - 1];
1810 
1811   for (i = len - 2; i >= 0; --i) {
1812     idx *= max[i];
1813     idx += tup[i];
1814   }
1815   return idx;
1816 }
1817 
1818 static PetscErrorCode DMPlexCreateHypercubicMesh_Internal(DM dm, PetscInt dim, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], const DMBoundaryType bd[])
1819 {
1820   Vec          coordinates;
1821   PetscSection coordSection;
1822   DMLabel      cutLabel    = NULL;
1823   PetscBool    cutMarker   = PETSC_FALSE;
1824   PetscBool    periodic    = PETSC_FALSE;
1825   PetscInt     numCells    = 1, c;
1826   PetscInt     numVertices = 1, v;
1827   PetscScalar *coords;
1828   PetscInt    *vertices, *vert, *vtmp, *supp, cone[2];
1829   PetscInt     d, e, cell = 0, coordSize;
1830   PetscMPIInt  rank;
1831 
1832   PetscFunctionBegin;
1833   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1834   PetscCall(DMSetDimension(dm, dim));
1835   PetscCall(PetscCalloc4(dim, &vertices, dim, &vert, dim, &vtmp, 2 * dim, &supp));
1836   PetscCall(DMCreateLabel(dm, "marker"));
1837   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
1838   for (d = 0; d < dim; ++d) periodic = (periodic || bd[d] == DM_BOUNDARY_PERIODIC) ? PETSC_TRUE : PETSC_FALSE;
1839   if (periodic && cutMarker) {
1840     PetscCall(DMCreateLabel(dm, "periodic_cut"));
1841     PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
1842   }
1843   for (d = 0; d < dim; ++d) PetscCheck(bd[d] == DM_BOUNDARY_PERIODIC, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Hypercubic mesh must be periodic now");
1844   for (d = 0; d < dim; ++d) {
1845     vertices[d] = edges[d];
1846     numVertices *= vertices[d];
1847   }
1848   numCells = numVertices * dim;
1849   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1850   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, 2));
1851   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetSupportSize(dm, v, 2 * dim));
1852   /* TODO Loop over boundary and reset support sizes */
1853   PetscCall(DMSetUp(dm)); /* Allocate space for cones and supports */
1854   /* Build cell cones and vertex supports */
1855   PetscCall(DMCreateLabel(dm, "celltype"));
1856   while (vert[dim - 1] < vertices[dim - 1]) {
1857     const PetscInt vertex = TupleToIndex_Private(dim, vertices, vert) + numCells;
1858     PetscInt       s      = 0;
1859 
1860     PetscCall(PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT ":", vertex));
1861     for (d = 0; d < dim; ++d) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, vert[d]));
1862     PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
1863     PetscCall(DMPlexSetCellType(dm, vertex, DM_POLYTOPE_POINT));
1864     for (d = 0; d < dim; ++d) {
1865       for (e = 0; e < dim; ++e) vtmp[e] = vert[e];
1866       vtmp[d] = (vert[d] + 1) % vertices[d];
1867       cone[0] = vertex;
1868       cone[1] = TupleToIndex_Private(dim, vertices, vtmp) + numCells;
1869       PetscCall(PetscPrintf(PETSC_COMM_SELF, "  Vertex %" PetscInt_FMT ":", cone[1]));
1870       for (e = 0; e < dim; ++e) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, vtmp[e]));
1871       PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
1872       PetscCall(DMPlexSetCone(dm, cell, cone));
1873       PetscCall(DMPlexSetCellType(dm, cell, DM_POLYTOPE_SEGMENT));
1874       PetscCall(PetscPrintf(PETSC_COMM_SELF, "  Edge %" PetscInt_FMT " (%" PetscInt_FMT " %" PetscInt_FMT ")\n", cell, cone[0], cone[1]));
1875       ++cell;
1876     }
1877     for (d = 0; d < dim; ++d) {
1878       for (e = 0; e < dim; ++e) vtmp[e] = vert[e];
1879       vtmp[d]   = (vert[d] + vertices[d] - 1) % vertices[d];
1880       supp[s++] = TupleToIndex_Private(dim, vertices, vtmp) * dim + d;
1881       supp[s++] = (vertex - numCells) * dim + d;
1882       PetscCall(DMPlexSetSupport(dm, vertex, supp));
1883     }
1884     PetscCall(DMPlexTensorPointLexicographic_Private(dim, vertices, vert));
1885   }
1886   PetscCall(DMPlexStratify(dm));
1887   /* Build coordinates */
1888   PetscCall(DMGetCoordinateSection(dm, &coordSection));
1889   PetscCall(PetscSectionSetNumFields(coordSection, 1));
1890   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1891   PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1892   for (v = numCells; v < numCells + numVertices; ++v) {
1893     PetscCall(PetscSectionSetDof(coordSection, v, dim));
1894     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1895   }
1896   PetscCall(PetscSectionSetUp(coordSection));
1897   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1898   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1899   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1900   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1901   PetscCall(VecSetBlockSize(coordinates, dim));
1902   PetscCall(VecSetType(coordinates, VECSTANDARD));
1903   PetscCall(VecGetArray(coordinates, &coords));
1904   for (d = 0; d < dim; ++d) vert[d] = 0;
1905   while (vert[dim - 1] < vertices[dim - 1]) {
1906     const PetscInt vertex = TupleToIndex_Private(dim, vertices, vert);
1907 
1908     for (d = 0; d < dim; ++d) coords[vertex * dim + d] = lower[d] + ((upper[d] - lower[d]) / vertices[d]) * vert[d];
1909     PetscCall(PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT ":", vertex));
1910     for (d = 0; d < dim; ++d) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, vert[d]));
1911     for (d = 0; d < dim; ++d) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %g", (double)PetscRealPart(coords[vertex * dim + d])));
1912     PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
1913     PetscCall(DMPlexTensorPointLexicographic_Private(dim, vertices, vert));
1914   }
1915   PetscCall(VecRestoreArray(coordinates, &coords));
1916   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1917   PetscCall(VecDestroy(&coordinates));
1918   PetscCall(PetscFree4(vertices, vert, vtmp, supp));
1919   //PetscCall(DMSetPeriodicity(dm, NULL, lower, upper));
1920   // Attach the extent
1921   {
1922     PetscContainer c;
1923     PetscInt      *extent;
1924 
1925     PetscCall(PetscMalloc1(dim, &extent));
1926     for (PetscInt d = 0; d < dim; ++d) extent[d] = edges[d];
1927     PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &c));
1928     PetscCall(PetscContainerSetCtxDestroy(c, PetscCtxDestroyDefault));
1929     PetscCall(PetscContainerSetPointer(c, extent));
1930     PetscCall(PetscObjectCompose((PetscObject)dm, "_extent", (PetscObject)c));
1931     PetscCall(PetscContainerDestroy(&c));
1932   }
1933   PetscFunctionReturn(PETSC_SUCCESS);
1934 }
1935 
1936 /*@C
1937   DMPlexCreateHypercubicMesh - Creates a periodic mesh on the tensor product of unit intervals using only vertices and edges.
1938 
1939   Collective
1940 
1941   Input Parameters:
1942 + comm  - The communicator for the DM object
1943 . dim   - The spatial dimension
1944 . edges - Number of edges per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1945 . lower - The lower left corner, or `NULL` for (0, 0, 0)
1946 - upper - The upper right corner, or `NULL` for (1, 1, 1)
1947 
1948   Output Parameter:
1949 . dm - The DM object
1950 
1951   Level: beginner
1952 
1953   Note:
1954   If you want to customize this mesh using options, you just need to
1955 .vb
1956   DMCreate(comm, &dm);
1957   DMSetType(dm, DMPLEX);
1958   DMSetFromOptions(dm);
1959 .ve
1960   and use the options on the `DMSetFromOptions()` page.
1961 
1962   The vertices are numbered is lexicographic order, and the dim edges exiting a vertex in the positive orthant are number consecutively,
1963 .vb
1964  18--0-19--2-20--4-18
1965   |     |     |     |
1966  13    15    17    13
1967   |     |     |     |
1968  24-12-25-14-26-16-24
1969   |     |     |     |
1970   7     9    11     7
1971   |     |     |     |
1972  21--6-22--8-23-10-21
1973   |     |     |     |
1974   1     3     5     1
1975   |     |     |     |
1976  18--0-19--2-20--4-18
1977 .ve
1978 
1979 .seealso: `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
1980 @*/
1981 PetscErrorCode DMPlexCreateHypercubicMesh(MPI_Comm comm, PetscInt dim, const PetscInt edges[], const PetscReal lower[], const PetscReal upper[], DM *dm)
1982 {
1983   PetscInt       *edg;
1984   PetscReal      *low, *upp;
1985   DMBoundaryType *bdt;
1986   PetscInt        d;
1987 
1988   PetscFunctionBegin;
1989   PetscCall(DMCreate(comm, dm));
1990   PetscCall(DMSetType(*dm, DMPLEX));
1991   PetscCall(PetscMalloc4(dim, &edg, dim, &low, dim, &upp, dim, &bdt));
1992   for (d = 0; d < dim; ++d) {
1993     edg[d] = edges ? edges[d] : 1;
1994     low[d] = lower ? lower[d] : 0.;
1995     upp[d] = upper ? upper[d] : 1.;
1996     bdt[d] = DM_BOUNDARY_PERIODIC;
1997   }
1998   PetscCall(DMPlexCreateHypercubicMesh_Internal(*dm, dim, low, upp, edg, bdt));
1999   PetscCall(PetscFree4(edg, low, upp, bdt));
2000   PetscFunctionReturn(PETSC_SUCCESS);
2001 }
2002 
2003 /*@
2004   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all `DM` options in the database.
2005 
2006   Logically Collective
2007 
2008   Input Parameters:
2009 + dm     - the `DM` context
2010 - prefix - the prefix to prepend to all option names
2011 
2012   Level: advanced
2013 
2014   Note:
2015   A hyphen (-) must NOT be given at the beginning of the prefix name.
2016   The first character of all runtime options is AUTOMATICALLY the hyphen.
2017 
2018 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `SNESSetFromOptions()`
2019 @*/
2020 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
2021 {
2022   DM_Plex *mesh = (DM_Plex *)dm->data;
2023 
2024   PetscFunctionBegin;
2025   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2026   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, prefix));
2027   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)mesh->partitioner, prefix));
2028   PetscFunctionReturn(PETSC_SUCCESS);
2029 }
2030 
2031 /* Remap geometry to cylinder
2032    TODO: This only works for a single refinement, then it is broken
2033 
2034      Interior square: Linear interpolation is correct
2035      The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
2036      such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
2037 
2038        phi     = arctan(y/x)
2039        d_close = sqrt(1/8 + 1/4 sin^2(phi))
2040        d_far   = sqrt(1/2 + sin^2(phi))
2041 
2042      so we remap them using
2043 
2044        x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
2045        y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
2046 
2047      If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
2048 */
2049 static void snapToCylinder(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
2050 {
2051   const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
2052   const PetscReal ds2 = 0.5 * dis;
2053 
2054   if ((PetscAbsScalar(u[0]) <= ds2) && (PetscAbsScalar(u[1]) <= ds2)) {
2055     f0[0] = u[0];
2056     f0[1] = u[1];
2057   } else {
2058     PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
2059 
2060     x    = PetscRealPart(u[0]);
2061     y    = PetscRealPart(u[1]);
2062     phi  = PetscAtan2Real(y, x);
2063     sinp = PetscSinReal(phi);
2064     cosp = PetscCosReal(phi);
2065     if ((PetscAbsReal(phi) > PETSC_PI / 4.0) && (PetscAbsReal(phi) < 3.0 * PETSC_PI / 4.0)) {
2066       dc = PetscAbsReal(ds2 / sinp);
2067       df = PetscAbsReal(dis / sinp);
2068       xc = ds2 * x / PetscAbsReal(y);
2069       yc = ds2 * PetscSignReal(y);
2070     } else {
2071       dc = PetscAbsReal(ds2 / cosp);
2072       df = PetscAbsReal(dis / cosp);
2073       xc = ds2 * PetscSignReal(x);
2074       yc = ds2 * y / PetscAbsReal(x);
2075     }
2076     f0[0] = xc + (u[0] - xc) * (1.0 - dc) / (df - dc);
2077     f0[1] = yc + (u[1] - yc) * (1.0 - dc) / (df - dc);
2078   }
2079   f0[2] = u[2];
2080 }
2081 
2082 static PetscErrorCode DMPlexCreateHexCylinderMesh_Internal(DM dm, DMBoundaryType periodicZ, PetscInt Nr)
2083 {
2084   const PetscInt dim = 3;
2085   PetscInt       numCells, numVertices;
2086   PetscMPIInt    rank;
2087 
2088   PetscFunctionBegin;
2089   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
2090   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2091   PetscCall(DMSetDimension(dm, dim));
2092   /* Create topology */
2093   {
2094     PetscInt cone[8], c;
2095 
2096     numCells    = rank == 0 ? 5 : 0;
2097     numVertices = rank == 0 ? 16 : 0;
2098     if (periodicZ == DM_BOUNDARY_PERIODIC) {
2099       numCells *= 3;
2100       numVertices = rank == 0 ? 24 : 0;
2101     }
2102     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
2103     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 8));
2104     PetscCall(DMSetUp(dm));
2105     if (rank == 0) {
2106       if (periodicZ == DM_BOUNDARY_PERIODIC) {
2107         cone[0] = 15;
2108         cone[1] = 18;
2109         cone[2] = 17;
2110         cone[3] = 16;
2111         cone[4] = 31;
2112         cone[5] = 32;
2113         cone[6] = 33;
2114         cone[7] = 34;
2115         PetscCall(DMPlexSetCone(dm, 0, cone));
2116         cone[0] = 16;
2117         cone[1] = 17;
2118         cone[2] = 24;
2119         cone[3] = 23;
2120         cone[4] = 32;
2121         cone[5] = 36;
2122         cone[6] = 37;
2123         cone[7] = 33; /* 22 25 26 21 */
2124         PetscCall(DMPlexSetCone(dm, 1, cone));
2125         cone[0] = 18;
2126         cone[1] = 27;
2127         cone[2] = 24;
2128         cone[3] = 17;
2129         cone[4] = 34;
2130         cone[5] = 33;
2131         cone[6] = 37;
2132         cone[7] = 38;
2133         PetscCall(DMPlexSetCone(dm, 2, cone));
2134         cone[0] = 29;
2135         cone[1] = 27;
2136         cone[2] = 18;
2137         cone[3] = 15;
2138         cone[4] = 35;
2139         cone[5] = 31;
2140         cone[6] = 34;
2141         cone[7] = 38;
2142         PetscCall(DMPlexSetCone(dm, 3, cone));
2143         cone[0] = 29;
2144         cone[1] = 15;
2145         cone[2] = 16;
2146         cone[3] = 23;
2147         cone[4] = 35;
2148         cone[5] = 36;
2149         cone[6] = 32;
2150         cone[7] = 31;
2151         PetscCall(DMPlexSetCone(dm, 4, cone));
2152 
2153         cone[0] = 31;
2154         cone[1] = 34;
2155         cone[2] = 33;
2156         cone[3] = 32;
2157         cone[4] = 19;
2158         cone[5] = 22;
2159         cone[6] = 21;
2160         cone[7] = 20;
2161         PetscCall(DMPlexSetCone(dm, 5, cone));
2162         cone[0] = 32;
2163         cone[1] = 33;
2164         cone[2] = 37;
2165         cone[3] = 36;
2166         cone[4] = 22;
2167         cone[5] = 25;
2168         cone[6] = 26;
2169         cone[7] = 21;
2170         PetscCall(DMPlexSetCone(dm, 6, cone));
2171         cone[0] = 34;
2172         cone[1] = 38;
2173         cone[2] = 37;
2174         cone[3] = 33;
2175         cone[4] = 20;
2176         cone[5] = 21;
2177         cone[6] = 26;
2178         cone[7] = 28;
2179         PetscCall(DMPlexSetCone(dm, 7, cone));
2180         cone[0] = 35;
2181         cone[1] = 38;
2182         cone[2] = 34;
2183         cone[3] = 31;
2184         cone[4] = 30;
2185         cone[5] = 19;
2186         cone[6] = 20;
2187         cone[7] = 28;
2188         PetscCall(DMPlexSetCone(dm, 8, cone));
2189         cone[0] = 35;
2190         cone[1] = 31;
2191         cone[2] = 32;
2192         cone[3] = 36;
2193         cone[4] = 30;
2194         cone[5] = 25;
2195         cone[6] = 22;
2196         cone[7] = 19;
2197         PetscCall(DMPlexSetCone(dm, 9, cone));
2198 
2199         cone[0] = 19;
2200         cone[1] = 20;
2201         cone[2] = 21;
2202         cone[3] = 22;
2203         cone[4] = 15;
2204         cone[5] = 16;
2205         cone[6] = 17;
2206         cone[7] = 18;
2207         PetscCall(DMPlexSetCone(dm, 10, cone));
2208         cone[0] = 22;
2209         cone[1] = 21;
2210         cone[2] = 26;
2211         cone[3] = 25;
2212         cone[4] = 16;
2213         cone[5] = 23;
2214         cone[6] = 24;
2215         cone[7] = 17;
2216         PetscCall(DMPlexSetCone(dm, 11, cone));
2217         cone[0] = 20;
2218         cone[1] = 28;
2219         cone[2] = 26;
2220         cone[3] = 21;
2221         cone[4] = 18;
2222         cone[5] = 17;
2223         cone[6] = 24;
2224         cone[7] = 27;
2225         PetscCall(DMPlexSetCone(dm, 12, cone));
2226         cone[0] = 30;
2227         cone[1] = 28;
2228         cone[2] = 20;
2229         cone[3] = 19;
2230         cone[4] = 29;
2231         cone[5] = 15;
2232         cone[6] = 18;
2233         cone[7] = 27;
2234         PetscCall(DMPlexSetCone(dm, 13, cone));
2235         cone[0] = 30;
2236         cone[1] = 19;
2237         cone[2] = 22;
2238         cone[3] = 25;
2239         cone[4] = 29;
2240         cone[5] = 23;
2241         cone[6] = 16;
2242         cone[7] = 15;
2243         PetscCall(DMPlexSetCone(dm, 14, cone));
2244       } else {
2245         cone[0] = 5;
2246         cone[1] = 8;
2247         cone[2] = 7;
2248         cone[3] = 6;
2249         cone[4] = 9;
2250         cone[5] = 12;
2251         cone[6] = 11;
2252         cone[7] = 10;
2253         PetscCall(DMPlexSetCone(dm, 0, cone));
2254         cone[0] = 6;
2255         cone[1] = 7;
2256         cone[2] = 14;
2257         cone[3] = 13;
2258         cone[4] = 12;
2259         cone[5] = 15;
2260         cone[6] = 16;
2261         cone[7] = 11;
2262         PetscCall(DMPlexSetCone(dm, 1, cone));
2263         cone[0] = 8;
2264         cone[1] = 17;
2265         cone[2] = 14;
2266         cone[3] = 7;
2267         cone[4] = 10;
2268         cone[5] = 11;
2269         cone[6] = 16;
2270         cone[7] = 18;
2271         PetscCall(DMPlexSetCone(dm, 2, cone));
2272         cone[0] = 19;
2273         cone[1] = 17;
2274         cone[2] = 8;
2275         cone[3] = 5;
2276         cone[4] = 20;
2277         cone[5] = 9;
2278         cone[6] = 10;
2279         cone[7] = 18;
2280         PetscCall(DMPlexSetCone(dm, 3, cone));
2281         cone[0] = 19;
2282         cone[1] = 5;
2283         cone[2] = 6;
2284         cone[3] = 13;
2285         cone[4] = 20;
2286         cone[5] = 15;
2287         cone[6] = 12;
2288         cone[7] = 9;
2289         PetscCall(DMPlexSetCone(dm, 4, cone));
2290       }
2291     }
2292     PetscCall(DMPlexSymmetrize(dm));
2293     PetscCall(DMPlexStratify(dm));
2294   }
2295   /* Create cube geometry */
2296   {
2297     Vec             coordinates;
2298     PetscSection    coordSection;
2299     PetscScalar    *coords;
2300     PetscInt        coordSize, v;
2301     const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
2302     const PetscReal ds2 = dis / 2.0;
2303 
2304     /* Build coordinates */
2305     PetscCall(DMGetCoordinateSection(dm, &coordSection));
2306     PetscCall(PetscSectionSetNumFields(coordSection, 1));
2307     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
2308     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
2309     for (v = numCells; v < numCells + numVertices; ++v) {
2310       PetscCall(PetscSectionSetDof(coordSection, v, dim));
2311       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
2312     }
2313     PetscCall(PetscSectionSetUp(coordSection));
2314     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2315     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2316     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2317     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2318     PetscCall(VecSetBlockSize(coordinates, dim));
2319     PetscCall(VecSetType(coordinates, VECSTANDARD));
2320     PetscCall(VecGetArray(coordinates, &coords));
2321     if (rank == 0) {
2322       coords[0 * dim + 0]  = -ds2;
2323       coords[0 * dim + 1]  = -ds2;
2324       coords[0 * dim + 2]  = 0.0;
2325       coords[1 * dim + 0]  = ds2;
2326       coords[1 * dim + 1]  = -ds2;
2327       coords[1 * dim + 2]  = 0.0;
2328       coords[2 * dim + 0]  = ds2;
2329       coords[2 * dim + 1]  = ds2;
2330       coords[2 * dim + 2]  = 0.0;
2331       coords[3 * dim + 0]  = -ds2;
2332       coords[3 * dim + 1]  = ds2;
2333       coords[3 * dim + 2]  = 0.0;
2334       coords[4 * dim + 0]  = -ds2;
2335       coords[4 * dim + 1]  = -ds2;
2336       coords[4 * dim + 2]  = 1.0;
2337       coords[5 * dim + 0]  = -ds2;
2338       coords[5 * dim + 1]  = ds2;
2339       coords[5 * dim + 2]  = 1.0;
2340       coords[6 * dim + 0]  = ds2;
2341       coords[6 * dim + 1]  = ds2;
2342       coords[6 * dim + 2]  = 1.0;
2343       coords[7 * dim + 0]  = ds2;
2344       coords[7 * dim + 1]  = -ds2;
2345       coords[7 * dim + 2]  = 1.0;
2346       coords[8 * dim + 0]  = dis;
2347       coords[8 * dim + 1]  = -dis;
2348       coords[8 * dim + 2]  = 0.0;
2349       coords[9 * dim + 0]  = dis;
2350       coords[9 * dim + 1]  = dis;
2351       coords[9 * dim + 2]  = 0.0;
2352       coords[10 * dim + 0] = dis;
2353       coords[10 * dim + 1] = -dis;
2354       coords[10 * dim + 2] = 1.0;
2355       coords[11 * dim + 0] = dis;
2356       coords[11 * dim + 1] = dis;
2357       coords[11 * dim + 2] = 1.0;
2358       coords[12 * dim + 0] = -dis;
2359       coords[12 * dim + 1] = dis;
2360       coords[12 * dim + 2] = 0.0;
2361       coords[13 * dim + 0] = -dis;
2362       coords[13 * dim + 1] = dis;
2363       coords[13 * dim + 2] = 1.0;
2364       coords[14 * dim + 0] = -dis;
2365       coords[14 * dim + 1] = -dis;
2366       coords[14 * dim + 2] = 0.0;
2367       coords[15 * dim + 0] = -dis;
2368       coords[15 * dim + 1] = -dis;
2369       coords[15 * dim + 2] = 1.0;
2370       if (periodicZ == DM_BOUNDARY_PERIODIC) {
2371         /* 15 31 19 */ coords[16 * dim + 0] = -ds2;
2372         coords[16 * dim + 1]                = -ds2;
2373         coords[16 * dim + 2]                = 0.5;
2374         /* 16 32 22 */ coords[17 * dim + 0] = ds2;
2375         coords[17 * dim + 1]                = -ds2;
2376         coords[17 * dim + 2]                = 0.5;
2377         /* 17 33 21 */ coords[18 * dim + 0] = ds2;
2378         coords[18 * dim + 1]                = ds2;
2379         coords[18 * dim + 2]                = 0.5;
2380         /* 18 34 20 */ coords[19 * dim + 0] = -ds2;
2381         coords[19 * dim + 1]                = ds2;
2382         coords[19 * dim + 2]                = 0.5;
2383         /* 29 35 30 */ coords[20 * dim + 0] = -dis;
2384         coords[20 * dim + 1]                = -dis;
2385         coords[20 * dim + 2]                = 0.5;
2386         /* 23 36 25 */ coords[21 * dim + 0] = dis;
2387         coords[21 * dim + 1]                = -dis;
2388         coords[21 * dim + 2]                = 0.5;
2389         /* 24 37 26 */ coords[22 * dim + 0] = dis;
2390         coords[22 * dim + 1]                = dis;
2391         coords[22 * dim + 2]                = 0.5;
2392         /* 27 38 28 */ coords[23 * dim + 0] = -dis;
2393         coords[23 * dim + 1]                = dis;
2394         coords[23 * dim + 2]                = 0.5;
2395       }
2396     }
2397     PetscCall(VecRestoreArray(coordinates, &coords));
2398     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2399     PetscCall(VecDestroy(&coordinates));
2400   }
2401   /* Create periodicity */
2402   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
2403     PetscReal L[3]       = {-1., -1., 0.};
2404     PetscReal maxCell[3] = {-1., -1., 0.};
2405     PetscReal lower[3]   = {0.0, 0.0, 0.0};
2406     PetscReal upper[3]   = {1.0, 1.0, 1.5};
2407     PetscInt  numZCells  = 3;
2408 
2409     L[2]       = upper[2] - lower[2];
2410     maxCell[2] = 1.1 * (L[2] / numZCells);
2411     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
2412   }
2413   {
2414     DM          cdm;
2415     PetscDS     cds;
2416     PetscScalar c[2] = {1.0, 1.0};
2417 
2418     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_TRUE, NULL));
2419     PetscCall(DMGetCoordinateDM(dm, &cdm));
2420     PetscCall(DMGetDS(cdm, &cds));
2421     PetscCall(PetscDSSetConstants(cds, 2, c));
2422   }
2423   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
2424 
2425   /* Wait for coordinate creation before doing in-place modification */
2426   PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2427 
2428   char        oldprefix[PETSC_MAX_PATH_LEN];
2429   const char *prefix;
2430 
2431   PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
2432   PetscCall(PetscStrncpy(oldprefix, prefix, PETSC_MAX_PATH_LEN));
2433   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, "petsc_cyl_ref_"));
2434   for (PetscInt r = 0; r < PetscMax(0, Nr); ++r) {
2435     DM rdm;
2436 
2437     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
2438     PetscCall(DMPlexReplace_Internal(dm, &rdm));
2439   }
2440   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, oldprefix));
2441   PetscCall(DMPlexRemapGeometry(dm, 0.0, snapToCylinder));
2442 
2443   DMLabel         bdlabel, edgelabel;
2444   IS              faceIS;
2445   const PetscInt *faces;
2446   PetscInt        Nf;
2447 
2448   PetscCall(DMCreateLabel(dm, "marker"));
2449   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
2450   PetscCall(DMCreateLabel(dm, "generatrix"));
2451   PetscCall(DMGetLabel(dm, "generatrix", &edgelabel));
2452   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
2453   // Remove faces on top and bottom
2454   PetscCall(DMLabelGetStratumIS(bdlabel, 1, &faceIS));
2455   if (faceIS) {
2456     PetscCall(ISGetLocalSize(faceIS, &Nf));
2457     PetscCall(ISGetIndices(faceIS, &faces));
2458     for (PetscInt f = 0; f < Nf; ++f) {
2459       PetscReal vol, normal[3];
2460 
2461       PetscCall(DMPlexComputeCellGeometryFVM(dm, faces[f], &vol, NULL, normal));
2462       if (PetscAbsReal(normal[2]) < PETSC_SMALL) PetscCall(DMLabelSetValue(edgelabel, faces[f], 1));
2463     }
2464     PetscCall(ISRestoreIndices(faceIS, &faces));
2465     PetscCall(ISDestroy(&faceIS));
2466   }
2467   PetscCall(DMPlexLabelComplete(dm, bdlabel));
2468   PetscCall(DMPlexLabelComplete(dm, edgelabel));
2469   PetscFunctionReturn(PETSC_SUCCESS);
2470 }
2471 
2472 /*@
2473   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
2474 
2475   Collective
2476 
2477   Input Parameters:
2478 + comm      - The communicator for the `DM` object
2479 . periodicZ - The boundary type for the Z direction
2480 - Nr        - The number of refinements to carry out
2481 
2482   Output Parameter:
2483 . dm - The `DM` object
2484 
2485   Level: beginner
2486 
2487   Note:
2488   Here is the output numbering looking from the bottom of the cylinder\:
2489 .vb
2490        17-----14
2491         |     |
2492         |  2  |
2493         |     |
2494  17-----8-----7-----14
2495   |     |     |     |
2496   |  3  |  0  |  1  |
2497   |     |     |     |
2498  19-----5-----6-----13
2499         |     |
2500         |  4  |
2501         |     |
2502        19-----13
2503 
2504  and up through the top
2505 
2506        18-----16
2507         |     |
2508         |  2  |
2509         |     |
2510  18----10----11-----16
2511   |     |     |     |
2512   |  3  |  0  |  1  |
2513   |     |     |     |
2514  20-----9----12-----15
2515         |     |
2516         |  4  |
2517         |     |
2518        20-----15
2519 .ve
2520 
2521 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2522 @*/
2523 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, DMBoundaryType periodicZ, PetscInt Nr, DM *dm)
2524 {
2525   PetscFunctionBegin;
2526   PetscAssertPointer(dm, 4);
2527   PetscCall(DMCreate(comm, dm));
2528   PetscCall(DMSetType(*dm, DMPLEX));
2529   PetscCall(DMPlexCreateHexCylinderMesh_Internal(*dm, periodicZ, Nr));
2530   PetscFunctionReturn(PETSC_SUCCESS);
2531 }
2532 
2533 static PetscErrorCode DMPlexCreateWedgeCylinderMesh_Internal(DM dm, PetscInt n, PetscBool interpolate)
2534 {
2535   const PetscInt dim = 3;
2536   PetscInt       numCells, numVertices, v;
2537   PetscMPIInt    rank;
2538 
2539   PetscFunctionBegin;
2540   PetscCheck(n >= 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %" PetscInt_FMT " cannot be negative", n);
2541   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
2542   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2543   PetscCall(DMSetDimension(dm, dim));
2544   /* Must create the celltype label here so that we do not automatically try to compute the types */
2545   PetscCall(DMCreateLabel(dm, "celltype"));
2546   /* Create topology */
2547   {
2548     PetscInt cone[6], c;
2549 
2550     numCells    = rank == 0 ? n : 0;
2551     numVertices = rank == 0 ? 2 * (n + 1) : 0;
2552     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
2553     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
2554     PetscCall(DMSetUp(dm));
2555     for (c = 0; c < numCells; c++) {
2556       cone[0] = c + n * 1;
2557       cone[1] = (c + 1) % n + n * 1;
2558       cone[2] = 0 + 3 * n;
2559       cone[3] = c + n * 2;
2560       cone[4] = (c + 1) % n + n * 2;
2561       cone[5] = 1 + 3 * n;
2562       PetscCall(DMPlexSetCone(dm, c, cone));
2563       PetscCall(DMPlexSetCellType(dm, c, DM_POLYTOPE_TRI_PRISM_TENSOR));
2564     }
2565     PetscCall(DMPlexSymmetrize(dm));
2566     PetscCall(DMPlexStratify(dm));
2567   }
2568   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetCellType(dm, v, DM_POLYTOPE_POINT));
2569   /* Create cylinder geometry */
2570   {
2571     Vec          coordinates;
2572     PetscSection coordSection;
2573     PetscScalar *coords;
2574     PetscInt     coordSize, c;
2575 
2576     /* Build coordinates */
2577     PetscCall(DMGetCoordinateSection(dm, &coordSection));
2578     PetscCall(PetscSectionSetNumFields(coordSection, 1));
2579     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
2580     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
2581     for (v = numCells; v < numCells + numVertices; ++v) {
2582       PetscCall(PetscSectionSetDof(coordSection, v, dim));
2583       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
2584     }
2585     PetscCall(PetscSectionSetUp(coordSection));
2586     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2587     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2588     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2589     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2590     PetscCall(VecSetBlockSize(coordinates, dim));
2591     PetscCall(VecSetType(coordinates, VECSTANDARD));
2592     PetscCall(VecGetArray(coordinates, &coords));
2593     for (c = 0; c < numCells; c++) {
2594       coords[(c + 0 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
2595       coords[(c + 0 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
2596       coords[(c + 0 * n) * dim + 2] = 1.0;
2597       coords[(c + 1 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
2598       coords[(c + 1 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
2599       coords[(c + 1 * n) * dim + 2] = 0.0;
2600     }
2601     if (rank == 0) {
2602       coords[(2 * n + 0) * dim + 0] = 0.0;
2603       coords[(2 * n + 0) * dim + 1] = 0.0;
2604       coords[(2 * n + 0) * dim + 2] = 1.0;
2605       coords[(2 * n + 1) * dim + 0] = 0.0;
2606       coords[(2 * n + 1) * dim + 1] = 0.0;
2607       coords[(2 * n + 1) * dim + 2] = 0.0;
2608     }
2609     PetscCall(VecRestoreArray(coordinates, &coords));
2610     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2611     PetscCall(VecDestroy(&coordinates));
2612   }
2613   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
2614   /* Interpolate */
2615   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2616   PetscFunctionReturn(PETSC_SUCCESS);
2617 }
2618 
2619 /*@
2620   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
2621 
2622   Collective
2623 
2624   Input Parameters:
2625 + comm        - The communicator for the `DM` object
2626 . n           - The number of wedges around the origin
2627 - interpolate - Create edges and faces
2628 
2629   Output Parameter:
2630 . dm - The `DM` object
2631 
2632   Level: beginner
2633 
2634 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2635 @*/
2636 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
2637 {
2638   PetscFunctionBegin;
2639   PetscAssertPointer(dm, 4);
2640   PetscCall(DMCreate(comm, dm));
2641   PetscCall(DMSetType(*dm, DMPLEX));
2642   PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(*dm, n, interpolate));
2643   PetscFunctionReturn(PETSC_SUCCESS);
2644 }
2645 
2646 static inline PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2647 {
2648   PetscReal prod = 0.0;
2649   PetscInt  i;
2650   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
2651   return PetscSqrtReal(prod);
2652 }
2653 
2654 static inline PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2655 {
2656   PetscReal prod = 0.0;
2657   PetscInt  i;
2658   for (i = 0; i < dim; ++i) prod += x[i] * y[i];
2659   return prod;
2660 }
2661 
2662 /* The first constant is the sphere radius */
2663 static void snapToSphere(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
2664 {
2665   PetscReal r     = PetscRealPart(constants[0]);
2666   PetscReal norm2 = 0.0, fac;
2667   PetscInt  n     = uOff[1] - uOff[0], d;
2668 
2669   for (d = 0; d < n; ++d) norm2 += PetscSqr(PetscRealPart(u[d]));
2670   fac = r / PetscSqrtReal(norm2);
2671   for (d = 0; d < n; ++d) f0[d] = u[d] * fac;
2672 }
2673 
2674 static PetscErrorCode DMPlexCreateSphereMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, PetscReal R)
2675 {
2676   const PetscInt embedDim = dim + 1;
2677   PetscSection   coordSection;
2678   Vec            coordinates;
2679   PetscScalar   *coords;
2680   PetscReal     *coordsIn;
2681   PetscInt       numCells, numEdges, numVerts = 0, firstVertex = 0, v, firstEdge, coordSize, d, e;
2682   PetscMPIInt    rank;
2683 
2684   PetscFunctionBegin;
2685   PetscValidLogicalCollectiveBool(dm, simplex, 3);
2686   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
2687   PetscCall(DMSetDimension(dm, dim));
2688   PetscCall(DMSetCoordinateDim(dm, dim + 1));
2689   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2690   switch (dim) {
2691   case 1:
2692     numCells = 16;
2693     numVerts = numCells;
2694 
2695     // Build Topology
2696     PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2697     for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2698     PetscCall(DMSetUp(dm));
2699     for (PetscInt c = 0; c < numCells; ++c) {
2700       PetscInt cone[2];
2701 
2702       cone[0] = c + numCells;
2703       cone[1] = (c + 1) % numVerts + numCells;
2704       PetscCall(DMPlexSetCone(dm, c, cone));
2705     }
2706     PetscCall(DMPlexSymmetrize(dm));
2707     PetscCall(DMPlexStratify(dm));
2708     PetscCall(PetscMalloc1(numVerts * embedDim, &coordsIn));
2709     for (PetscInt v = 0; v < numVerts; ++v) {
2710       const PetscReal rad = 2. * PETSC_PI * v / numVerts;
2711 
2712       coordsIn[v * embedDim + 0] = PetscCosReal(rad);
2713       coordsIn[v * embedDim + 1] = PetscSinReal(rad);
2714     }
2715     break;
2716   case 2:
2717     if (simplex) {
2718       const PetscReal radius    = PetscSqrtReal(1 + PETSC_PHI * PETSC_PHI) / (1.0 + PETSC_PHI);
2719       const PetscReal edgeLen   = 2.0 / (1.0 + PETSC_PHI) * (R / radius);
2720       const PetscInt  degree    = 5;
2721       PetscReal       vertex[3] = {0.0, 1.0 / (1.0 + PETSC_PHI), PETSC_PHI / (1.0 + PETSC_PHI)};
2722       PetscInt        s[3]      = {1, 1, 1};
2723       PetscInt        cone[3];
2724       PetscInt       *graph;
2725 
2726       vertex[0] *= R / radius;
2727       vertex[1] *= R / radius;
2728       vertex[2] *= R / radius;
2729       numCells    = rank == 0 ? 20 : 0;
2730       numVerts    = rank == 0 ? 12 : 0;
2731       firstVertex = numCells;
2732       /* Use icosahedron, which for a R-sphere has coordinates which are all cyclic permutations of
2733 
2734            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
2735 
2736          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2737          length is then given by 2/(1+\phi) = 2 * 0.38197 = 0.76393.
2738       */
2739       /* Construct vertices */
2740       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2741       if (rank == 0) {
2742         for (PetscInt p = 0, i = 0; p < embedDim; ++p) {
2743           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2744             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2745               for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertex[(d + p) % embedDim];
2746               ++i;
2747             }
2748           }
2749         }
2750       }
2751       /* Construct graph */
2752       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2753       for (PetscInt i = 0; i < numVerts; ++i) {
2754         PetscInt k = 0;
2755         for (PetscInt j = 0; j < numVerts; ++j) {
2756           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2757             graph[i * numVerts + j] = 1;
2758             ++k;
2759           }
2760         }
2761         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid icosahedron, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2762       }
2763       /* Build Topology */
2764       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2765       for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2766       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2767       /* Cells */
2768       for (PetscInt i = 0, c = 0; i < numVerts; ++i) {
2769         for (PetscInt j = 0; j < i; ++j) {
2770           for (PetscInt k = 0; k < j; ++k) {
2771             if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i]) {
2772               cone[0] = firstVertex + i;
2773               cone[1] = firstVertex + j;
2774               cone[2] = firstVertex + k;
2775               /* Check orientation */
2776               {
2777                 const PetscInt epsilon[3][3][3] = {
2778                   {{0, 0, 0},  {0, 0, 1},  {0, -1, 0}},
2779                   {{0, 0, -1}, {0, 0, 0},  {1, 0, 0} },
2780                   {{0, 1, 0},  {-1, 0, 0}, {0, 0, 0} }
2781                 };
2782                 PetscReal normal[3];
2783                 PetscInt  e, f;
2784 
2785                 for (d = 0; d < embedDim; ++d) {
2786                   normal[d] = 0.0;
2787                   for (e = 0; e < embedDim; ++e) {
2788                     for (f = 0; f < embedDim; ++f) normal[d] += epsilon[d][e][f] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]);
2789                   }
2790                 }
2791                 if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2792                   PetscInt tmp = cone[1];
2793                   cone[1]      = cone[2];
2794                   cone[2]      = tmp;
2795                 }
2796               }
2797               PetscCall(DMPlexSetCone(dm, c++, cone));
2798             }
2799           }
2800         }
2801       }
2802       PetscCall(DMPlexSymmetrize(dm));
2803       PetscCall(DMPlexStratify(dm));
2804       PetscCall(PetscFree(graph));
2805     } else {
2806       /*
2807         12-21--13
2808          |     |
2809         25  4  24
2810          |     |
2811   12-25--9-16--8-24--13
2812    |     |     |     |
2813   23  5 17  0 15  3  22
2814    |     |     |     |
2815   10-20--6-14--7-19--11
2816          |     |
2817         20  1  19
2818          |     |
2819         10-18--11
2820          |     |
2821         23  2  22
2822          |     |
2823         12-21--13
2824        */
2825       PetscInt cone[4], ornt[4];
2826 
2827       numCells    = rank == 0 ? 6 : 0;
2828       numEdges    = rank == 0 ? 12 : 0;
2829       numVerts    = rank == 0 ? 8 : 0;
2830       firstVertex = numCells;
2831       firstEdge   = numCells + numVerts;
2832       /* Build Topology */
2833       PetscCall(DMPlexSetChart(dm, 0, numCells + numEdges + numVerts));
2834       for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 4));
2835       for (e = firstEdge; e < firstEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
2836       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2837       if (rank == 0) {
2838         /* Cell 0 */
2839         cone[0] = 14;
2840         cone[1] = 15;
2841         cone[2] = 16;
2842         cone[3] = 17;
2843         PetscCall(DMPlexSetCone(dm, 0, cone));
2844         ornt[0] = 0;
2845         ornt[1] = 0;
2846         ornt[2] = 0;
2847         ornt[3] = 0;
2848         PetscCall(DMPlexSetConeOrientation(dm, 0, ornt));
2849         /* Cell 1 */
2850         cone[0] = 18;
2851         cone[1] = 19;
2852         cone[2] = 14;
2853         cone[3] = 20;
2854         PetscCall(DMPlexSetCone(dm, 1, cone));
2855         ornt[0] = 0;
2856         ornt[1] = 0;
2857         ornt[2] = -1;
2858         ornt[3] = 0;
2859         PetscCall(DMPlexSetConeOrientation(dm, 1, ornt));
2860         /* Cell 2 */
2861         cone[0] = 21;
2862         cone[1] = 22;
2863         cone[2] = 18;
2864         cone[3] = 23;
2865         PetscCall(DMPlexSetCone(dm, 2, cone));
2866         ornt[0] = 0;
2867         ornt[1] = 0;
2868         ornt[2] = -1;
2869         ornt[3] = 0;
2870         PetscCall(DMPlexSetConeOrientation(dm, 2, ornt));
2871         /* Cell 3 */
2872         cone[0] = 19;
2873         cone[1] = 22;
2874         cone[2] = 24;
2875         cone[3] = 15;
2876         PetscCall(DMPlexSetCone(dm, 3, cone));
2877         ornt[0] = -1;
2878         ornt[1] = -1;
2879         ornt[2] = 0;
2880         ornt[3] = -1;
2881         PetscCall(DMPlexSetConeOrientation(dm, 3, ornt));
2882         /* Cell 4 */
2883         cone[0] = 16;
2884         cone[1] = 24;
2885         cone[2] = 21;
2886         cone[3] = 25;
2887         PetscCall(DMPlexSetCone(dm, 4, cone));
2888         ornt[0] = -1;
2889         ornt[1] = -1;
2890         ornt[2] = -1;
2891         ornt[3] = 0;
2892         PetscCall(DMPlexSetConeOrientation(dm, 4, ornt));
2893         /* Cell 5 */
2894         cone[0] = 20;
2895         cone[1] = 17;
2896         cone[2] = 25;
2897         cone[3] = 23;
2898         PetscCall(DMPlexSetCone(dm, 5, cone));
2899         ornt[0] = -1;
2900         ornt[1] = -1;
2901         ornt[2] = -1;
2902         ornt[3] = -1;
2903         PetscCall(DMPlexSetConeOrientation(dm, 5, ornt));
2904         /* Edges */
2905         cone[0] = 6;
2906         cone[1] = 7;
2907         PetscCall(DMPlexSetCone(dm, 14, cone));
2908         cone[0] = 7;
2909         cone[1] = 8;
2910         PetscCall(DMPlexSetCone(dm, 15, cone));
2911         cone[0] = 8;
2912         cone[1] = 9;
2913         PetscCall(DMPlexSetCone(dm, 16, cone));
2914         cone[0] = 9;
2915         cone[1] = 6;
2916         PetscCall(DMPlexSetCone(dm, 17, cone));
2917         cone[0] = 10;
2918         cone[1] = 11;
2919         PetscCall(DMPlexSetCone(dm, 18, cone));
2920         cone[0] = 11;
2921         cone[1] = 7;
2922         PetscCall(DMPlexSetCone(dm, 19, cone));
2923         cone[0] = 6;
2924         cone[1] = 10;
2925         PetscCall(DMPlexSetCone(dm, 20, cone));
2926         cone[0] = 12;
2927         cone[1] = 13;
2928         PetscCall(DMPlexSetCone(dm, 21, cone));
2929         cone[0] = 13;
2930         cone[1] = 11;
2931         PetscCall(DMPlexSetCone(dm, 22, cone));
2932         cone[0] = 10;
2933         cone[1] = 12;
2934         PetscCall(DMPlexSetCone(dm, 23, cone));
2935         cone[0] = 13;
2936         cone[1] = 8;
2937         PetscCall(DMPlexSetCone(dm, 24, cone));
2938         cone[0] = 12;
2939         cone[1] = 9;
2940         PetscCall(DMPlexSetCone(dm, 25, cone));
2941       }
2942       PetscCall(DMPlexSymmetrize(dm));
2943       PetscCall(DMPlexStratify(dm));
2944       /* Build coordinates */
2945       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2946       if (rank == 0) {
2947         coordsIn[0 * embedDim + 0] = -R;
2948         coordsIn[0 * embedDim + 1] = R;
2949         coordsIn[0 * embedDim + 2] = -R;
2950         coordsIn[1 * embedDim + 0] = R;
2951         coordsIn[1 * embedDim + 1] = R;
2952         coordsIn[1 * embedDim + 2] = -R;
2953         coordsIn[2 * embedDim + 0] = R;
2954         coordsIn[2 * embedDim + 1] = -R;
2955         coordsIn[2 * embedDim + 2] = -R;
2956         coordsIn[3 * embedDim + 0] = -R;
2957         coordsIn[3 * embedDim + 1] = -R;
2958         coordsIn[3 * embedDim + 2] = -R;
2959         coordsIn[4 * embedDim + 0] = -R;
2960         coordsIn[4 * embedDim + 1] = R;
2961         coordsIn[4 * embedDim + 2] = R;
2962         coordsIn[5 * embedDim + 0] = R;
2963         coordsIn[5 * embedDim + 1] = R;
2964         coordsIn[5 * embedDim + 2] = R;
2965         coordsIn[6 * embedDim + 0] = -R;
2966         coordsIn[6 * embedDim + 1] = -R;
2967         coordsIn[6 * embedDim + 2] = R;
2968         coordsIn[7 * embedDim + 0] = R;
2969         coordsIn[7 * embedDim + 1] = -R;
2970         coordsIn[7 * embedDim + 2] = R;
2971       }
2972     }
2973     break;
2974   case 3:
2975     if (simplex) {
2976       const PetscReal edgeLen         = 1.0 / PETSC_PHI;
2977       PetscReal       vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
2978       PetscReal       vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
2979       PetscReal       vertexC[4]      = {0.5, 0.5 * PETSC_PHI, 0.5 / PETSC_PHI, 0.0};
2980       const PetscInt  degree          = 12;
2981       PetscInt        s[4]            = {1, 1, 1};
2982       PetscInt        evenPerm[12][4] = {
2983         {0, 1, 2, 3},
2984         {0, 2, 3, 1},
2985         {0, 3, 1, 2},
2986         {1, 0, 3, 2},
2987         {1, 2, 0, 3},
2988         {1, 3, 2, 0},
2989         {2, 0, 1, 3},
2990         {2, 1, 3, 0},
2991         {2, 3, 0, 1},
2992         {3, 0, 2, 1},
2993         {3, 1, 0, 2},
2994         {3, 2, 1, 0}
2995       };
2996       PetscInt  cone[4];
2997       PetscInt *graph, p, i, j, k, l;
2998 
2999       vertexA[0] *= R;
3000       vertexA[1] *= R;
3001       vertexA[2] *= R;
3002       vertexA[3] *= R;
3003       vertexB[0] *= R;
3004       vertexB[1] *= R;
3005       vertexB[2] *= R;
3006       vertexB[3] *= R;
3007       vertexC[0] *= R;
3008       vertexC[1] *= R;
3009       vertexC[2] *= R;
3010       vertexC[3] *= R;
3011       numCells    = rank == 0 ? 600 : 0;
3012       numVerts    = rank == 0 ? 120 : 0;
3013       firstVertex = numCells;
3014       /* Use the 600-cell, which for a unit sphere has coordinates which are
3015 
3016            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
3017                (\pm 1,    0,       0,      0)  all cyclic permutations   8
3018            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
3019 
3020          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
3021          length is then given by 1/\phi = 0.61803.
3022 
3023          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
3024          http://mathworld.wolfram.com/600-Cell.html
3025       */
3026       /* Construct vertices */
3027       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
3028       i = 0;
3029       if (rank == 0) {
3030         for (s[0] = -1; s[0] < 2; s[0] += 2) {
3031           for (s[1] = -1; s[1] < 2; s[1] += 2) {
3032             for (s[2] = -1; s[2] < 2; s[2] += 2) {
3033               for (s[3] = -1; s[3] < 2; s[3] += 2) {
3034                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[d] * vertexA[d];
3035                 ++i;
3036               }
3037             }
3038           }
3039         }
3040         for (p = 0; p < embedDim; ++p) {
3041           s[1] = s[2] = s[3] = 1;
3042           for (s[0] = -1; s[0] < 2; s[0] += 2) {
3043             for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertexB[(d + p) % embedDim];
3044             ++i;
3045           }
3046         }
3047         for (p = 0; p < 12; ++p) {
3048           s[3] = 1;
3049           for (s[0] = -1; s[0] < 2; s[0] += 2) {
3050             for (s[1] = -1; s[1] < 2; s[1] += 2) {
3051               for (s[2] = -1; s[2] < 2; s[2] += 2) {
3052                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[evenPerm[p][d]] * vertexC[evenPerm[p][d]];
3053                 ++i;
3054               }
3055             }
3056           }
3057         }
3058       }
3059       PetscCheck(i == numVerts, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertices %" PetscInt_FMT " != %" PetscInt_FMT, i, numVerts);
3060       /* Construct graph */
3061       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
3062       for (i = 0; i < numVerts; ++i) {
3063         for (j = 0, k = 0; j < numVerts; ++j) {
3064           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
3065             graph[i * numVerts + j] = 1;
3066             ++k;
3067           }
3068         }
3069         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
3070       }
3071       /* Build Topology */
3072       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
3073       for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
3074       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
3075       /* Cells */
3076       if (rank == 0) {
3077         for (PetscInt i = 0, c = 0; i < numVerts; ++i) {
3078           for (j = 0; j < i; ++j) {
3079             for (k = 0; k < j; ++k) {
3080               for (l = 0; l < k; ++l) {
3081                 if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i] && graph[l * numVerts + i] && graph[l * numVerts + j] && graph[l * numVerts + k]) {
3082                   cone[0] = firstVertex + i;
3083                   cone[1] = firstVertex + j;
3084                   cone[2] = firstVertex + k;
3085                   cone[3] = firstVertex + l;
3086                   /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
3087                   {
3088                     const PetscInt epsilon[4][4][4][4] = {
3089                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, -1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 0, 0}, {0, 1, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 1, 0}, {0, -1, 0, 0}, {0, 0, 0, 0}}},
3090 
3091                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}}, {{0, 0, -1, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}}},
3092 
3093                       {{{0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, 0, 0}, {0, -1, 0, 0}}, {{0, 0, 0, -1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 1, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}},
3094 
3095                       {{{0, 0, 0, 0}, {0, 0, -1, 0}, {0, 1, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 1, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}}, {{0, -1, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}} }
3096                     };
3097                     PetscReal normal[4];
3098                     PetscInt  e, f, g;
3099 
3100                     for (d = 0; d < embedDim; ++d) {
3101                       normal[d] = 0.0;
3102                       for (e = 0; e < embedDim; ++e) {
3103                         for (f = 0; f < embedDim; ++f) {
3104                           for (g = 0; g < embedDim; ++g) {
3105                             normal[d] += epsilon[d][e][f][g] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]) * (coordsIn[l * embedDim + f] - coordsIn[i * embedDim + f]);
3106                           }
3107                         }
3108                       }
3109                     }
3110                     if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
3111                       PetscInt tmp = cone[1];
3112                       cone[1]      = cone[2];
3113                       cone[2]      = tmp;
3114                     }
3115                   }
3116                   PetscCall(DMPlexSetCone(dm, c++, cone));
3117                 }
3118               }
3119             }
3120           }
3121         }
3122       }
3123       PetscCall(DMPlexSymmetrize(dm));
3124       PetscCall(DMPlexStratify(dm));
3125       PetscCall(PetscFree(graph));
3126     }
3127     break;
3128   default:
3129     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unsupported dimension for sphere: %" PetscInt_FMT, dim);
3130   }
3131   /* Create coordinates */
3132   PetscCall(DMGetCoordinateSection(dm, &coordSection));
3133   PetscCall(PetscSectionSetNumFields(coordSection, 1));
3134   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, embedDim));
3135   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVerts));
3136   for (v = firstVertex; v < firstVertex + numVerts; ++v) {
3137     PetscCall(PetscSectionSetDof(coordSection, v, embedDim));
3138     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, embedDim));
3139   }
3140   PetscCall(PetscSectionSetUp(coordSection));
3141   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
3142   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
3143   PetscCall(VecSetBlockSize(coordinates, embedDim));
3144   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
3145   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
3146   PetscCall(VecSetType(coordinates, VECSTANDARD));
3147   PetscCall(VecGetArray(coordinates, &coords));
3148   for (v = 0; v < numVerts; ++v)
3149     for (d = 0; d < embedDim; ++d) coords[v * embedDim + d] = coordsIn[v * embedDim + d];
3150   PetscCall(VecRestoreArray(coordinates, &coords));
3151   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
3152   PetscCall(VecDestroy(&coordinates));
3153   PetscCall(PetscFree(coordsIn));
3154   {
3155     DM          cdm;
3156     PetscDS     cds;
3157     PetscScalar c = R;
3158 
3159     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_TRUE, snapToSphere));
3160     PetscCall(DMGetCoordinateDM(dm, &cdm));
3161     PetscCall(DMGetDS(cdm, &cds));
3162     PetscCall(PetscDSSetConstants(cds, 1, &c));
3163   }
3164   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
3165   /* Wait for coordinate creation before doing in-place modification */
3166   if (simplex) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
3167   PetscFunctionReturn(PETSC_SUCCESS);
3168 }
3169 
3170 typedef void (*TPSEvaluateFunc)(const PetscReal[], PetscReal *, PetscReal[], PetscReal (*)[3]);
3171 
3172 /*
3173  The Schwarz P implicit surface is
3174 
3175      f(x) = cos(x0) + cos(x1) + cos(x2) = 0
3176 */
3177 static void TPSEvaluate_SchwarzP(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
3178 {
3179   PetscReal c[3] = {PetscCosReal(y[0] * PETSC_PI), PetscCosReal(y[1] * PETSC_PI), PetscCosReal(y[2] * PETSC_PI)};
3180   PetscReal g[3] = {-PetscSinReal(y[0] * PETSC_PI), -PetscSinReal(y[1] * PETSC_PI), -PetscSinReal(y[2] * PETSC_PI)};
3181   f[0]           = c[0] + c[1] + c[2];
3182   for (PetscInt i = 0; i < 3; i++) {
3183     grad[i] = PETSC_PI * g[i];
3184     for (PetscInt j = 0; j < 3; j++) hess[i][j] = (i == j) ? -PetscSqr(PETSC_PI) * c[i] : 0.;
3185   }
3186 }
3187 
3188 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
3189 static PetscErrorCode TPSExtrudeNormalFunc_SchwarzP(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
3190 {
3191   for (PetscInt i = 0; i < 3; i++) u[i] = -PETSC_PI * PetscSinReal(x[i] * PETSC_PI);
3192   return PETSC_SUCCESS;
3193 }
3194 
3195 /*
3196  The Gyroid implicit surface is
3197 
3198  f(x,y,z) = sin(pi * x) * cos (pi * (y + 1/2))  + sin(pi * (y + 1/2)) * cos(pi * (z + 1/4)) + sin(pi * (z + 1/4)) * cos(pi * x)
3199 
3200 */
3201 static void TPSEvaluate_Gyroid(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
3202 {
3203   PetscReal s[3] = {PetscSinReal(PETSC_PI * y[0]), PetscSinReal(PETSC_PI * (y[1] + .5)), PetscSinReal(PETSC_PI * (y[2] + .25))};
3204   PetscReal c[3] = {PetscCosReal(PETSC_PI * y[0]), PetscCosReal(PETSC_PI * (y[1] + .5)), PetscCosReal(PETSC_PI * (y[2] + .25))};
3205   f[0]           = s[0] * c[1] + s[1] * c[2] + s[2] * c[0];
3206   grad[0]        = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
3207   grad[1]        = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
3208   grad[2]        = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
3209   hess[0][0]     = -PetscSqr(PETSC_PI) * (s[0] * c[1] + s[2] * c[0]);
3210   hess[0][1]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
3211   hess[0][2]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
3212   hess[1][0]     = -PetscSqr(PETSC_PI) * (s[1] * c[2] + s[0] * c[1]);
3213   hess[1][1]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
3214   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
3215   hess[2][0]     = -PetscSqr(PETSC_PI) * (s[2] * c[0] + s[1] * c[2]);
3216   hess[2][1]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
3217   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
3218 }
3219 
3220 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
3221 static PetscErrorCode TPSExtrudeNormalFunc_Gyroid(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
3222 {
3223   PetscReal s[3] = {PetscSinReal(PETSC_PI * x[0]), PetscSinReal(PETSC_PI * (x[1] + .5)), PetscSinReal(PETSC_PI * (x[2] + .25))};
3224   PetscReal c[3] = {PetscCosReal(PETSC_PI * x[0]), PetscCosReal(PETSC_PI * (x[1] + .5)), PetscCosReal(PETSC_PI * (x[2] + .25))};
3225   u[0]           = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
3226   u[1]           = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
3227   u[2]           = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
3228   return PETSC_SUCCESS;
3229 }
3230 
3231 /*
3232    We wish to solve
3233 
3234          min_y || y - x ||^2  subject to f(y) = 0
3235 
3236    Let g(y) = grad(f).  The minimization problem is equivalent to asking to satisfy
3237    f(y) = 0 and (y-x) is parallel to g(y).  We do this by using Householder QR to obtain a basis for the
3238    tangent space and ask for both components in the tangent space to be zero.
3239 
3240    Take g to be a column vector and compute the "full QR" factorization Q R = g,
3241    where Q = I - 2 n n^T is a symmetric orthogonal matrix.
3242    The first column of Q is parallel to g so the remaining two columns span the null space.
3243    Let Qn = Q[:,1:] be those remaining columns.  Then Qn Qn^T is an orthogonal projector into the tangent space.
3244    Since Q is symmetric, this is equivalent to multiplying by Q and taking the last two entries.
3245    In total, we have a system of 3 equations in 3 unknowns:
3246 
3247      f(y) = 0                       1 equation
3248      Qn^T (y - x) = 0               2 equations
3249 
3250    Here, we compute the residual and Jacobian of this system.
3251 */
3252 static void TPSNearestPointResJac(TPSEvaluateFunc feval, const PetscScalar x[], const PetscScalar y[], PetscScalar res[], PetscScalar J[])
3253 {
3254   PetscReal yreal[3] = {PetscRealPart(y[0]), PetscRealPart(y[1]), PetscRealPart(y[2])};
3255   PetscReal d[3]     = {PetscRealPart(y[0] - x[0]), PetscRealPart(y[1] - x[1]), PetscRealPart(y[2] - x[2])};
3256   PetscReal f, grad[3], n[3], norm, norm_y[3], nd, nd_y[3], sign;
3257   PetscReal n_y[3][3] = {
3258     {0, 0, 0},
3259     {0, 0, 0},
3260     {0, 0, 0}
3261   };
3262 
3263   feval(yreal, &f, grad, n_y);
3264 
3265   for (PetscInt i = 0; i < 3; i++) n[i] = grad[i];
3266   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
3267   for (PetscInt i = 0; i < 3; i++) norm_y[i] = 1. / norm * n[i] * n_y[i][i];
3268 
3269   // Define the Householder reflector
3270   sign = n[0] >= 0 ? 1. : -1.;
3271   n[0] += norm * sign;
3272   for (PetscInt i = 0; i < 3; i++) n_y[0][i] += norm_y[i] * sign;
3273 
3274   norm      = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
3275   norm_y[0] = 1. / norm * (n[0] * n_y[0][0]);
3276   norm_y[1] = 1. / norm * (n[0] * n_y[0][1] + n[1] * n_y[1][1]);
3277   norm_y[2] = 1. / norm * (n[0] * n_y[0][2] + n[2] * n_y[2][2]);
3278 
3279   for (PetscInt i = 0; i < 3; i++) {
3280     n[i] /= norm;
3281     for (PetscInt j = 0; j < 3; j++) {
3282       // note that n[i] is n_old[i]/norm when executing the code below
3283       n_y[i][j] = n_y[i][j] / norm - n[i] / norm * norm_y[j];
3284     }
3285   }
3286 
3287   nd = n[0] * d[0] + n[1] * d[1] + n[2] * d[2];
3288   for (PetscInt i = 0; i < 3; i++) nd_y[i] = n[i] + n_y[0][i] * d[0] + n_y[1][i] * d[1] + n_y[2][i] * d[2];
3289 
3290   res[0] = f;
3291   res[1] = d[1] - 2 * n[1] * nd;
3292   res[2] = d[2] - 2 * n[2] * nd;
3293   // J[j][i] is J_{ij} (column major)
3294   for (PetscInt j = 0; j < 3; j++) {
3295     J[0 + j * 3] = grad[j];
3296     J[1 + j * 3] = (j == 1) * 1. - 2 * (n_y[1][j] * nd + n[1] * nd_y[j]);
3297     J[2 + j * 3] = (j == 2) * 1. - 2 * (n_y[2][j] * nd + n[2] * nd_y[j]);
3298   }
3299 }
3300 
3301 /*
3302    Project x to the nearest point on the implicit surface using Newton's method.
3303 */
3304 static PetscErrorCode TPSNearestPoint(TPSEvaluateFunc feval, PetscScalar x[])
3305 {
3306   PetscScalar y[3] = {x[0], x[1], x[2]}; // Initial guess
3307 
3308   PetscFunctionBegin;
3309   for (PetscInt iter = 0; iter < 10; iter++) {
3310     PetscScalar res[3], J[9];
3311     PetscReal   resnorm;
3312     TPSNearestPointResJac(feval, x, y, res, J);
3313     resnorm = PetscSqrtReal(PetscSqr(PetscRealPart(res[0])) + PetscSqr(PetscRealPart(res[1])) + PetscSqr(PetscRealPart(res[2])));
3314     if (0) { // Turn on this monitor if you need to confirm quadratic convergence
3315       PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%" PetscInt_FMT "] res [%g %g %g]\n", iter, (double)PetscRealPart(res[0]), (double)PetscRealPart(res[1]), (double)PetscRealPart(res[2])));
3316     }
3317     if (resnorm < PETSC_SMALL) break;
3318 
3319     // Take the Newton step
3320     PetscCall(PetscKernel_A_gets_inverse_A_3(J, 0., PETSC_FALSE, NULL));
3321     PetscKernel_v_gets_v_minus_A_times_w_3(y, J, res);
3322   }
3323   for (PetscInt i = 0; i < 3; i++) x[i] = y[i];
3324   PetscFunctionReturn(PETSC_SUCCESS);
3325 }
3326 
3327 const char *const DMPlexTPSTypes[] = {"SCHWARZ_P", "GYROID", "DMPlexTPSType", "DMPLEX_TPS_", NULL};
3328 
3329 static PetscErrorCode DMPlexCreateTPSMesh_Internal(DM dm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness)
3330 {
3331   PetscMPIInt rank;
3332   PetscInt    topoDim = 2, spaceDim = 3, numFaces = 0, numVertices = 0, numEdges = 0;
3333   PetscInt(*edges)[2] = NULL, *edgeSets = NULL;
3334   PetscInt           *cells_flat = NULL;
3335   PetscReal          *vtxCoords  = NULL;
3336   TPSEvaluateFunc     evalFunc   = NULL;
3337   PetscSimplePointFn *normalFunc = NULL;
3338   DMLabel             label;
3339 
3340   PetscFunctionBegin;
3341   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
3342   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3343   PetscCheck((layers != 0) ^ (thickness == 0.), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Layers %" PetscInt_FMT " must be nonzero iff thickness %g is nonzero", layers, (double)thickness);
3344   switch (tpstype) {
3345   case DMPLEX_TPS_SCHWARZ_P:
3346     PetscCheck(!periodic || (periodic[0] == DM_BOUNDARY_NONE && periodic[1] == DM_BOUNDARY_NONE && periodic[2] == DM_BOUNDARY_NONE), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Schwarz P does not support periodic meshes");
3347     if (rank == 0) {
3348       PetscInt(*cells)[6][4][4] = NULL; // [junction, junction-face, cell, conn]
3349       PetscInt  Njunctions = 0, Ncuts = 0, Npipes[3], vcount;
3350       PetscReal L = 1;
3351 
3352       Npipes[0]   = (extent[0] + 1) * extent[1] * extent[2];
3353       Npipes[1]   = extent[0] * (extent[1] + 1) * extent[2];
3354       Npipes[2]   = extent[0] * extent[1] * (extent[2] + 1);
3355       Njunctions  = extent[0] * extent[1] * extent[2];
3356       Ncuts       = 2 * (extent[0] * extent[1] + extent[1] * extent[2] + extent[2] * extent[0]);
3357       numVertices = 4 * (Npipes[0] + Npipes[1] + Npipes[2]) + 8 * Njunctions;
3358       PetscCall(PetscMalloc1(3 * numVertices, &vtxCoords));
3359       PetscCall(PetscMalloc1(Njunctions, &cells));
3360       PetscCall(PetscMalloc1(Ncuts * 4, &edges));
3361       PetscCall(PetscMalloc1(Ncuts * 4, &edgeSets));
3362       // x-normal pipes
3363       vcount = 0;
3364       for (PetscInt i = 0; i < extent[0] + 1; i++) {
3365         for (PetscInt j = 0; j < extent[1]; j++) {
3366           for (PetscInt k = 0; k < extent[2]; k++) {
3367             for (PetscInt l = 0; l < 4; l++) {
3368               vtxCoords[vcount++] = (2 * i - 1) * L;
3369               vtxCoords[vcount++] = 2 * j * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3370               vtxCoords[vcount++] = 2 * k * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3371             }
3372           }
3373         }
3374       }
3375       // y-normal pipes
3376       for (PetscInt i = 0; i < extent[0]; i++) {
3377         for (PetscInt j = 0; j < extent[1] + 1; j++) {
3378           for (PetscInt k = 0; k < extent[2]; k++) {
3379             for (PetscInt l = 0; l < 4; l++) {
3380               vtxCoords[vcount++] = 2 * i * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3381               vtxCoords[vcount++] = (2 * j - 1) * L;
3382               vtxCoords[vcount++] = 2 * k * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3383             }
3384           }
3385         }
3386       }
3387       // z-normal pipes
3388       for (PetscInt i = 0; i < extent[0]; i++) {
3389         for (PetscInt j = 0; j < extent[1]; j++) {
3390           for (PetscInt k = 0; k < extent[2] + 1; k++) {
3391             for (PetscInt l = 0; l < 4; l++) {
3392               vtxCoords[vcount++] = 2 * i * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3393               vtxCoords[vcount++] = 2 * j * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3394               vtxCoords[vcount++] = (2 * k - 1) * L;
3395             }
3396           }
3397         }
3398       }
3399       // junctions
3400       for (PetscInt i = 0; i < extent[0]; i++) {
3401         for (PetscInt j = 0; j < extent[1]; j++) {
3402           for (PetscInt k = 0; k < extent[2]; k++) {
3403             const PetscInt J = (i * extent[1] + j) * extent[2] + k, Jvoff = (Npipes[0] + Npipes[1] + Npipes[2]) * 4 + J * 8;
3404             PetscCheck(vcount / 3 == Jvoff, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected vertex count");
3405             for (PetscInt ii = 0; ii < 2; ii++) {
3406               for (PetscInt jj = 0; jj < 2; jj++) {
3407                 for (PetscInt kk = 0; kk < 2; kk++) {
3408                   double Ls           = (1 - sqrt(2) / 4) * L;
3409                   vtxCoords[vcount++] = 2 * i * L + (2 * ii - 1) * Ls;
3410                   vtxCoords[vcount++] = 2 * j * L + (2 * jj - 1) * Ls;
3411                   vtxCoords[vcount++] = 2 * k * L + (2 * kk - 1) * Ls;
3412                 }
3413               }
3414             }
3415             const PetscInt jfaces[3][2][4] = {
3416               {{3, 1, 0, 2}, {7, 5, 4, 6}}, // x-aligned
3417               {{5, 4, 0, 1}, {7, 6, 2, 3}}, // y-aligned
3418               {{6, 2, 0, 4}, {7, 3, 1, 5}}  // z-aligned
3419             };
3420             const PetscInt pipe_lo[3] = {// vertex numbers of pipes
3421                                          ((i * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + Npipes[0] + Npipes[1]) * 4};
3422             const PetscInt pipe_hi[3] = {// vertex numbers of pipes
3423                                          (((i + 1) * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j + 1) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + 1 + Npipes[0] + Npipes[1]) * 4};
3424             for (PetscInt dir = 0; dir < 3; dir++) { // x,y,z
3425               const PetscInt ijk[3] = {i, j, k};
3426               for (PetscInt l = 0; l < 4; l++) { // rotations
3427                 cells[J][dir * 2 + 0][l][0] = pipe_lo[dir] + l;
3428                 cells[J][dir * 2 + 0][l][1] = Jvoff + jfaces[dir][0][l];
3429                 cells[J][dir * 2 + 0][l][2] = Jvoff + jfaces[dir][0][(l - 1 + 4) % 4];
3430                 cells[J][dir * 2 + 0][l][3] = pipe_lo[dir] + (l - 1 + 4) % 4;
3431                 cells[J][dir * 2 + 1][l][0] = Jvoff + jfaces[dir][1][l];
3432                 cells[J][dir * 2 + 1][l][1] = pipe_hi[dir] + l;
3433                 cells[J][dir * 2 + 1][l][2] = pipe_hi[dir] + (l - 1 + 4) % 4;
3434                 cells[J][dir * 2 + 1][l][3] = Jvoff + jfaces[dir][1][(l - 1 + 4) % 4];
3435                 if (ijk[dir] == 0) {
3436                   edges[numEdges][0] = pipe_lo[dir] + l;
3437                   edges[numEdges][1] = pipe_lo[dir] + (l + 1) % 4;
3438                   edgeSets[numEdges] = dir * 2 + 1;
3439                   numEdges++;
3440                 }
3441                 if (ijk[dir] + 1 == extent[dir]) {
3442                   edges[numEdges][0] = pipe_hi[dir] + l;
3443                   edges[numEdges][1] = pipe_hi[dir] + (l + 1) % 4;
3444                   edgeSets[numEdges] = dir * 2 + 2;
3445                   numEdges++;
3446                 }
3447               }
3448             }
3449           }
3450         }
3451       }
3452       PetscCheck(numEdges == Ncuts * 4, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Edge count %" PetscInt_FMT " incompatible with number of cuts %" PetscInt_FMT, numEdges, Ncuts);
3453       numFaces   = 24 * Njunctions;
3454       cells_flat = cells[0][0][0];
3455     }
3456     evalFunc   = TPSEvaluate_SchwarzP;
3457     normalFunc = TPSExtrudeNormalFunc_SchwarzP;
3458     break;
3459   case DMPLEX_TPS_GYROID:
3460     if (rank == 0) {
3461       // This is a coarse mesh approximation of the gyroid shifted to being the zero of the level set
3462       //
3463       //     sin(pi*x)*cos(pi*(y+1/2)) + sin(pi*(y+1/2))*cos(pi*(z+1/4)) + sin(pi*(z+1/4))*cos(x)
3464       //
3465       // on the cell [0,2]^3.
3466       //
3467       // Think about dividing that cell into four columns, and focus on the column [0,1]x[0,1]x[0,2].
3468       // If you looked at the gyroid in that column at different slices of z you would see that it kind of spins
3469       // like a boomerang:
3470       //
3471       //     z = 0          z = 1/4        z = 1/2        z = 3/4     //
3472       //     -----          -------        -------        -------     //
3473       //                                                              //
3474       //     +       +      +       +      +       +      +   \   +   //
3475       //      \                                   /            \      //
3476       //       \            `-_   _-'            /              }     //
3477       //        *-_            `-'            _-'              /      //
3478       //     +     `-+      +       +      +-'     +      +   /   +   //
3479       //                                                              //
3480       //                                                              //
3481       //     z = 1          z = 5/4        z = 3/2        z = 7/4     //
3482       //     -----          -------        -------        -------     //
3483       //                                                              //
3484       //     +-_     +      +       +      +     _-+      +   /   +   //
3485       //        `-_            _-_            _-`            /        //
3486       //           \        _-'   `-_        /              {         //
3487       //            \                       /                \        //
3488       //     +       +      +       +      +       +      +   \   +   //
3489       //
3490       //
3491       // This course mesh approximates each of these slices by two line segments,
3492       // and then connects the segments in consecutive layers with quadrilateral faces.
3493       // All of the end points of the segments are multiples of 1/4 except for the
3494       // point * in the picture for z = 0 above and the similar points in other layers.
3495       // That point is at (gamma, gamma, 0), where gamma is calculated below.
3496       //
3497       // The column  [1,2]x[1,2]x[0,2] looks the same as this column;
3498       // The columns [1,2]x[0,1]x[0,2] and [0,1]x[1,2]x[0,2] are mirror images.
3499       //
3500       // As for how this method turned into the names given to the vertices:
3501       // that was not systematic, it was just the way it worked out in my handwritten notes.
3502 
3503       PetscInt facesPerBlock = 64;
3504       PetscInt vertsPerBlock = 56;
3505       PetscInt extentPlus[3];
3506       PetscInt numBlocks, numBlocksPlus;
3507       const PetscInt A = 0, B = 1, C = 2, D = 3, E = 4, F = 5, G = 6, H = 7, II = 8, J = 9, K = 10, L = 11, M = 12, N = 13, O = 14, P = 15, Q = 16, R = 17, S = 18, T = 19, U = 20, V = 21, W = 22, X = 23, Y = 24, Z = 25, Ap = 26, Bp = 27, Cp = 28, Dp = 29, Ep = 30, Fp = 31, Gp = 32, Hp = 33, Ip = 34, Jp = 35, Kp = 36, Lp = 37, Mp = 38, Np = 39, Op = 40, Pp = 41, Qp = 42, Rp = 43, Sp = 44, Tp = 45, Up = 46, Vp = 47, Wp = 48, Xp = 49, Yp = 50, Zp = 51, Aq = 52, Bq = 53, Cq = 54, Dq = 55;
3508       const PetscInt pattern[64][4] = {
3509         /* face to vertex within the coarse discretization of a single gyroid block */
3510         /* layer 0 */
3511         {A,           C,           K,           G          },
3512         {C,           B,           II,          K          },
3513         {D,           A,           H,           L          },
3514         {B + 56 * 1,  D,           L,           J          },
3515         {E,           B + 56 * 1,  J,           N          },
3516         {A + 56 * 2,  E,           N,           H + 56 * 2 },
3517         {F,           A + 56 * 2,  G + 56 * 2,  M          },
3518         {B,           F,           M,           II         },
3519         /* layer 1 */
3520         {G,           K,           Q,           O          },
3521         {K,           II,          P,           Q          },
3522         {L,           H,           O + 56 * 1,  R          },
3523         {J,           L,           R,           P          },
3524         {N,           J,           P,           S          },
3525         {H + 56 * 2,  N,           S,           O + 56 * 3 },
3526         {M,           G + 56 * 2,  O + 56 * 2,  T          },
3527         {II,          M,           T,           P          },
3528         /* layer 2 */
3529         {O,           Q,           Y,           U          },
3530         {Q,           P,           W,           Y          },
3531         {R,           O + 56 * 1,  U + 56 * 1,  Ap         },
3532         {P,           R,           Ap,          W          },
3533         {S,           P,           X,           Bp         },
3534         {O + 56 * 3,  S,           Bp,          V + 56 * 1 },
3535         {T,           O + 56 * 2,  V,           Z          },
3536         {P,           T,           Z,           X          },
3537         /* layer 3 */
3538         {U,           Y,           Ep,          Dp         },
3539         {Y,           W,           Cp,          Ep         },
3540         {Ap,          U + 56 * 1,  Dp + 56 * 1, Gp         },
3541         {W,           Ap,          Gp,          Cp         },
3542         {Bp,          X,           Cp + 56 * 2, Fp         },
3543         {V + 56 * 1,  Bp,          Fp,          Dp + 56 * 1},
3544         {Z,           V,           Dp,          Hp         },
3545         {X,           Z,           Hp,          Cp + 56 * 2},
3546         /* layer 4 */
3547         {Dp,          Ep,          Mp,          Kp         },
3548         {Ep,          Cp,          Ip,          Mp         },
3549         {Gp,          Dp + 56 * 1, Lp,          Np         },
3550         {Cp,          Gp,          Np,          Jp         },
3551         {Fp,          Cp + 56 * 2, Jp + 56 * 2, Pp         },
3552         {Dp + 56 * 1, Fp,          Pp,          Lp         },
3553         {Hp,          Dp,          Kp,          Op         },
3554         {Cp + 56 * 2, Hp,          Op,          Ip + 56 * 2},
3555         /* layer 5 */
3556         {Kp,          Mp,          Sp,          Rp         },
3557         {Mp,          Ip,          Qp,          Sp         },
3558         {Np,          Lp,          Rp,          Tp         },
3559         {Jp,          Np,          Tp,          Qp + 56 * 1},
3560         {Pp,          Jp + 56 * 2, Qp + 56 * 3, Up         },
3561         {Lp,          Pp,          Up,          Rp         },
3562         {Op,          Kp,          Rp,          Vp         },
3563         {Ip + 56 * 2, Op,          Vp,          Qp + 56 * 2},
3564         /* layer 6 */
3565         {Rp,          Sp,          Aq,          Yp         },
3566         {Sp,          Qp,          Wp,          Aq         },
3567         {Tp,          Rp,          Yp,          Cq         },
3568         {Qp + 56 * 1, Tp,          Cq,          Wp + 56 * 1},
3569         {Up,          Qp + 56 * 3, Xp + 56 * 1, Dq         },
3570         {Rp,          Up,          Dq,          Zp         },
3571         {Vp,          Rp,          Zp,          Bq         },
3572         {Qp + 56 * 2, Vp,          Bq,          Xp         },
3573         /* layer 7 (the top is the periodic image of the bottom of layer 0) */
3574         {Yp,          Aq,          C + 56 * 4,  A + 56 * 4 },
3575         {Aq,          Wp,          B + 56 * 4,  C + 56 * 4 },
3576         {Cq,          Yp,          A + 56 * 4,  D + 56 * 4 },
3577         {Wp + 56 * 1, Cq,          D + 56 * 4,  B + 56 * 5 },
3578         {Dq,          Xp + 56 * 1, B + 56 * 5,  E + 56 * 4 },
3579         {Zp,          Dq,          E + 56 * 4,  A + 56 * 6 },
3580         {Bq,          Zp,          A + 56 * 6,  F + 56 * 4 },
3581         {Xp,          Bq,          F + 56 * 4,  B + 56 * 4 }
3582       };
3583       const PetscReal gamma                = PetscAcosReal((PetscSqrtReal(3.) - 1.) / PetscSqrtReal(2.)) / PETSC_PI;
3584       const PetscReal patternCoords[56][3] = {
3585         {1.,        0.,        0.  }, /* A  */
3586         {0.,        1.,        0.  }, /* B  */
3587         {gamma,     gamma,     0.  }, /* C  */
3588         {1 + gamma, 1 - gamma, 0.  }, /* D  */
3589         {2 - gamma, 2 - gamma, 0.  }, /* E  */
3590         {1 - gamma, 1 + gamma, 0.  }, /* F  */
3591 
3592         {.5,        0,         .25 }, /* G  */
3593         {1.5,       0.,        .25 }, /* H  */
3594         {.5,        1.,        .25 }, /* II */
3595         {1.5,       1.,        .25 }, /* J  */
3596         {.25,       .5,        .25 }, /* K  */
3597         {1.25,      .5,        .25 }, /* L  */
3598         {.75,       1.5,       .25 }, /* M  */
3599         {1.75,      1.5,       .25 }, /* N  */
3600 
3601         {0.,        0.,        .5  }, /* O  */
3602         {1.,        1.,        .5  }, /* P  */
3603         {gamma,     1 - gamma, .5  }, /* Q  */
3604         {1 + gamma, gamma,     .5  }, /* R  */
3605         {2 - gamma, 1 + gamma, .5  }, /* S  */
3606         {1 - gamma, 2 - gamma, .5  }, /* T  */
3607 
3608         {0.,        .5,        .75 }, /* U  */
3609         {0.,        1.5,       .75 }, /* V  */
3610         {1.,        .5,        .75 }, /* W  */
3611         {1.,        1.5,       .75 }, /* X  */
3612         {.5,        .75,       .75 }, /* Y  */
3613         {.5,        1.75,      .75 }, /* Z  */
3614         {1.5,       .25,       .75 }, /* Ap */
3615         {1.5,       1.25,      .75 }, /* Bp */
3616 
3617         {1.,        0.,        1.  }, /* Cp */
3618         {0.,        1.,        1.  }, /* Dp */
3619         {1 - gamma, 1 - gamma, 1.  }, /* Ep */
3620         {1 + gamma, 1 + gamma, 1.  }, /* Fp */
3621         {2 - gamma, gamma,     1.  }, /* Gp */
3622         {gamma,     2 - gamma, 1.  }, /* Hp */
3623 
3624         {.5,        0.,        1.25}, /* Ip */
3625         {1.5,       0.,        1.25}, /* Jp */
3626         {.5,        1.,        1.25}, /* Kp */
3627         {1.5,       1.,        1.25}, /* Lp */
3628         {.75,       .5,        1.25}, /* Mp */
3629         {1.75,      .5,        1.25}, /* Np */
3630         {.25,       1.5,       1.25}, /* Op */
3631         {1.25,      1.5,       1.25}, /* Pp */
3632 
3633         {0.,        0.,        1.5 }, /* Qp */
3634         {1.,        1.,        1.5 }, /* Rp */
3635         {1 - gamma, gamma,     1.5 }, /* Sp */
3636         {2 - gamma, 1 - gamma, 1.5 }, /* Tp */
3637         {1 + gamma, 2 - gamma, 1.5 }, /* Up */
3638         {gamma,     1 + gamma, 1.5 }, /* Vp */
3639 
3640         {0.,        .5,        1.75}, /* Wp */
3641         {0.,        1.5,       1.75}, /* Xp */
3642         {1.,        .5,        1.75}, /* Yp */
3643         {1.,        1.5,       1.75}, /* Zp */
3644         {.5,        .25,       1.75}, /* Aq */
3645         {.5,        1.25,      1.75}, /* Bq */
3646         {1.5,       .75,       1.75}, /* Cq */
3647         {1.5,       1.75,      1.75}, /* Dq */
3648       };
3649       PetscInt(*cells)[64][4] = NULL;
3650       PetscBool *seen;
3651       PetscInt  *vertToTrueVert;
3652       PetscInt   count;
3653 
3654       for (PetscInt i = 0; i < 3; i++) extentPlus[i] = extent[i] + 1;
3655       numBlocks = 1;
3656       for (PetscInt i = 0; i < 3; i++) numBlocks *= extent[i];
3657       numBlocksPlus = 1;
3658       for (PetscInt i = 0; i < 3; i++) numBlocksPlus *= extentPlus[i];
3659       numFaces = numBlocks * facesPerBlock;
3660       PetscCall(PetscMalloc1(numBlocks, &cells));
3661       PetscCall(PetscCalloc1(numBlocksPlus * vertsPerBlock, &seen));
3662       for (PetscInt k = 0; k < extent[2]; k++) {
3663         for (PetscInt j = 0; j < extent[1]; j++) {
3664           for (PetscInt i = 0; i < extent[0]; i++) {
3665             for (PetscInt f = 0; f < facesPerBlock; f++) {
3666               for (PetscInt v = 0; v < 4; v++) {
3667                 PetscInt vertRaw     = pattern[f][v];
3668                 PetscInt blockidx    = vertRaw / 56;
3669                 PetscInt patternvert = vertRaw % 56;
3670                 PetscInt xplus       = (blockidx & 1);
3671                 PetscInt yplus       = (blockidx & 2) >> 1;
3672                 PetscInt zplus       = (blockidx & 4) >> 2;
3673                 PetscInt zcoord      = (periodic && periodic[2] == DM_BOUNDARY_PERIODIC) ? ((k + zplus) % extent[2]) : (k + zplus);
3674                 PetscInt ycoord      = (periodic && periodic[1] == DM_BOUNDARY_PERIODIC) ? ((j + yplus) % extent[1]) : (j + yplus);
3675                 PetscInt xcoord      = (periodic && periodic[0] == DM_BOUNDARY_PERIODIC) ? ((i + xplus) % extent[0]) : (i + xplus);
3676                 PetscInt vert        = ((zcoord * extentPlus[1] + ycoord) * extentPlus[0] + xcoord) * 56 + patternvert;
3677 
3678                 cells[(k * extent[1] + j) * extent[0] + i][f][v] = vert;
3679                 seen[vert]                                       = PETSC_TRUE;
3680               }
3681             }
3682           }
3683         }
3684       }
3685       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++)
3686         if (seen[i]) numVertices++;
3687       count = 0;
3688       PetscCall(PetscMalloc1(numBlocksPlus * vertsPerBlock, &vertToTrueVert));
3689       PetscCall(PetscMalloc1(numVertices * 3, &vtxCoords));
3690       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) vertToTrueVert[i] = -1;
3691       for (PetscInt k = 0; k < extentPlus[2]; k++) {
3692         for (PetscInt j = 0; j < extentPlus[1]; j++) {
3693           for (PetscInt i = 0; i < extentPlus[0]; i++) {
3694             for (PetscInt v = 0; v < vertsPerBlock; v++) {
3695               PetscInt vIdx = ((k * extentPlus[1] + j) * extentPlus[0] + i) * vertsPerBlock + v;
3696 
3697               if (seen[vIdx]) {
3698                 PetscInt thisVert;
3699 
3700                 vertToTrueVert[vIdx] = thisVert = count++;
3701 
3702                 for (PetscInt d = 0; d < 3; d++) vtxCoords[3 * thisVert + d] = patternCoords[v][d];
3703                 vtxCoords[3 * thisVert + 0] += i * 2;
3704                 vtxCoords[3 * thisVert + 1] += j * 2;
3705                 vtxCoords[3 * thisVert + 2] += k * 2;
3706               }
3707             }
3708           }
3709         }
3710       }
3711       for (PetscInt i = 0; i < numBlocks; i++) {
3712         for (PetscInt f = 0; f < facesPerBlock; f++) {
3713           for (PetscInt v = 0; v < 4; v++) cells[i][f][v] = vertToTrueVert[cells[i][f][v]];
3714         }
3715       }
3716       PetscCall(PetscFree(vertToTrueVert));
3717       PetscCall(PetscFree(seen));
3718       cells_flat = cells[0][0];
3719       numEdges   = 0;
3720       for (PetscInt i = 0; i < numFaces; i++) {
3721         for (PetscInt e = 0; e < 4; e++) {
3722           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3723           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3724 
3725           for (PetscInt d = 0; d < 3; d++) {
3726             if (!periodic || periodic[0] != DM_BOUNDARY_PERIODIC) {
3727               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) numEdges++;
3728               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) numEdges++;
3729             }
3730           }
3731         }
3732       }
3733       PetscCall(PetscMalloc1(numEdges, &edges));
3734       PetscCall(PetscMalloc1(numEdges, &edgeSets));
3735       for (PetscInt edge = 0, i = 0; i < numFaces; i++) {
3736         for (PetscInt e = 0; e < 4; e++) {
3737           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3738           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3739 
3740           for (PetscInt d = 0; d < 3; d++) {
3741             if (!periodic || periodic[d] != DM_BOUNDARY_PERIODIC) {
3742               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) {
3743                 edges[edge][0]   = ev[0];
3744                 edges[edge][1]   = ev[1];
3745                 edgeSets[edge++] = 2 * d;
3746               }
3747               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) {
3748                 edges[edge][0]   = ev[0];
3749                 edges[edge][1]   = ev[1];
3750                 edgeSets[edge++] = 2 * d + 1;
3751               }
3752             }
3753           }
3754         }
3755       }
3756     }
3757     evalFunc   = TPSEvaluate_Gyroid;
3758     normalFunc = TPSExtrudeNormalFunc_Gyroid;
3759     break;
3760   }
3761 
3762   PetscCall(DMSetDimension(dm, topoDim));
3763   if (rank == 0) PetscCall(DMPlexBuildFromCellList(dm, numFaces, numVertices, 4, cells_flat));
3764   else PetscCall(DMPlexBuildFromCellList(dm, 0, 0, 0, NULL));
3765   PetscCall(PetscFree(cells_flat));
3766   {
3767     DM idm;
3768     PetscCall(DMPlexInterpolate(dm, &idm));
3769     PetscCall(DMPlexReplace_Internal(dm, &idm));
3770   }
3771   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, vtxCoords));
3772   else PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, NULL));
3773   PetscCall(PetscFree(vtxCoords));
3774 
3775   PetscCall(DMCreateLabel(dm, "Face Sets"));
3776   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3777   for (PetscInt e = 0; e < numEdges; e++) {
3778     PetscInt        njoin;
3779     const PetscInt *join, verts[] = {numFaces + edges[e][0], numFaces + edges[e][1]};
3780     PetscCall(DMPlexGetJoin(dm, 2, verts, &njoin, &join));
3781     PetscCheck(njoin == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected unique join of vertices %" PetscInt_FMT " and %" PetscInt_FMT, edges[e][0], edges[e][1]);
3782     PetscCall(DMLabelSetValue(label, join[0], edgeSets[e]));
3783     PetscCall(DMPlexRestoreJoin(dm, 2, verts, &njoin, &join));
3784   }
3785   PetscCall(PetscFree(edges));
3786   PetscCall(PetscFree(edgeSets));
3787   if (tps_distribute) {
3788     DM               pdm = NULL;
3789     PetscPartitioner part;
3790 
3791     PetscCall(DMPlexGetPartitioner(dm, &part));
3792     PetscCall(PetscPartitionerSetFromOptions(part));
3793     PetscCall(DMPlexDistribute(dm, 0, NULL, &pdm));
3794     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
3795     // Do not auto-distribute again
3796     PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
3797   }
3798 
3799   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3800   for (PetscInt refine = 0; refine < refinements; refine++) {
3801     PetscInt     m;
3802     DM           dmf;
3803     Vec          X;
3804     PetscScalar *x;
3805     PetscCall(DMRefine(dm, MPI_COMM_NULL, &dmf));
3806     PetscCall(DMPlexReplace_Internal(dm, &dmf));
3807 
3808     PetscCall(DMGetCoordinatesLocal(dm, &X));
3809     PetscCall(VecGetLocalSize(X, &m));
3810     PetscCall(VecGetArray(X, &x));
3811     for (PetscInt i = 0; i < m; i += 3) PetscCall(TPSNearestPoint(evalFunc, &x[i]));
3812     PetscCall(VecRestoreArray(X, &x));
3813   }
3814 
3815   // Face Sets has already been propagated to new vertices during refinement; this propagates to the initial vertices.
3816   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3817   PetscCall(DMPlexLabelComplete(dm, label));
3818 
3819   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
3820 
3821   if (thickness > 0) {
3822     DM              edm, cdm, ecdm;
3823     DMPlexTransform tr;
3824     const char     *prefix;
3825     PetscOptions    options;
3826     // Code from DMPlexExtrude
3827     PetscCall(DMPlexTransformCreate(PetscObjectComm((PetscObject)dm), &tr));
3828     PetscCall(DMPlexTransformSetDM(tr, dm));
3829     PetscCall(DMPlexTransformSetType(tr, DMPLEXEXTRUDE));
3830     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
3831     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)tr, prefix));
3832     PetscCall(PetscObjectGetOptions((PetscObject)dm, &options));
3833     PetscCall(PetscObjectSetOptions((PetscObject)tr, options));
3834     PetscCall(DMPlexTransformExtrudeSetLayers(tr, layers));
3835     PetscCall(DMPlexTransformExtrudeSetThickness(tr, thickness));
3836     PetscCall(DMPlexTransformExtrudeSetTensor(tr, PETSC_FALSE));
3837     PetscCall(DMPlexTransformExtrudeSetSymmetric(tr, PETSC_TRUE));
3838     PetscCall(DMPlexTransformExtrudeSetNormalFunction(tr, normalFunc));
3839     PetscCall(DMPlexTransformSetFromOptions(tr));
3840     PetscCall(PetscObjectSetOptions((PetscObject)tr, NULL));
3841     PetscCall(DMPlexTransformSetUp(tr));
3842     PetscCall(PetscObjectViewFromOptions((PetscObject)tr, NULL, "-dm_plex_tps_transform_view"));
3843     PetscCall(DMPlexTransformApply(tr, dm, &edm));
3844     PetscCall(DMCopyDisc(dm, edm));
3845     PetscCall(DMGetCoordinateDM(dm, &cdm));
3846     PetscCall(DMGetCoordinateDM(edm, &ecdm));
3847     PetscCall(DMCopyDisc(cdm, ecdm));
3848     PetscCall(DMPlexTransformCreateDiscLabels(tr, edm));
3849     PetscCall(DMPlexTransformDestroy(&tr));
3850     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, edm));
3851     PetscCall(DMPlexReplace_Internal(dm, &edm));
3852   }
3853   PetscFunctionReturn(PETSC_SUCCESS);
3854 }
3855 
3856 /*@
3857   DMPlexCreateTPSMesh - Create a distributed, interpolated mesh of a triply-periodic surface
3858 
3859   Collective
3860 
3861   Input Parameters:
3862 + comm           - The communicator for the `DM` object
3863 . tpstype        - Type of triply-periodic surface
3864 . extent         - Array of length 3 containing number of periods in each direction
3865 . periodic       - array of length 3 with periodicity, or `NULL` for non-periodic
3866 . tps_distribute - Distribute 2D manifold mesh prior to refinement and extrusion (more scalable)
3867 . refinements    - Number of factor-of-2 refinements of 2D manifold mesh
3868 . layers         - Number of cell layers extruded in normal direction
3869 - thickness      - Thickness in normal direction
3870 
3871   Output Parameter:
3872 . dm - The `DM` object
3873 
3874   Level: beginner
3875 
3876   Notes:
3877   This meshes the surface of the Schwarz P or Gyroid surfaces.  Schwarz P is the simplest member of the triply-periodic minimal surfaces.
3878   <https://en.wikipedia.org/wiki/Schwarz_minimal_surface#Schwarz_P_(%22Primitive%22)> and can be cut with "clean" boundaries.
3879   The Gyroid <https://en.wikipedia.org/wiki/Gyroid> is another triply-periodic minimal surface with applications in additive manufacturing; it is much more difficult to "cut" since there are no planes of symmetry.
3880   Our implementation creates a very coarse mesh of the surface and refines (by 4-way splitting) as many times as requested.
3881   On each refinement, all vertices are projected to their nearest point on the surface.
3882   This projection could readily be extended to related surfaces.
3883 
3884   See {cite}`maskery2018insights`
3885 
3886   The face (edge) sets for the Schwarz P surface are numbered $1(-x), 2(+x), 3(-y), 4(+y), 5(-z), 6(+z)$.
3887   When the mesh is refined, "Face Sets" contain the new vertices (created during refinement).
3888   Use `DMPlexLabelComplete()` to propagate to coarse-level vertices.
3889 
3890   Developer Notes:
3891   The Gyroid mesh does not currently mark boundary sets.
3892 
3893 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMSetType()`, `DMCreate()`
3894 @*/
3895 PetscErrorCode DMPlexCreateTPSMesh(MPI_Comm comm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness, DM *dm)
3896 {
3897   PetscFunctionBegin;
3898   PetscCall(DMCreate(comm, dm));
3899   PetscCall(DMSetType(*dm, DMPLEX));
3900   PetscCall(DMPlexCreateTPSMesh_Internal(*dm, tpstype, extent, periodic, tps_distribute, refinements, layers, thickness));
3901   PetscFunctionReturn(PETSC_SUCCESS);
3902 }
3903 
3904 /*@
3905   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
3906 
3907   Collective
3908 
3909   Input Parameters:
3910 + comm    - The communicator for the `DM` object
3911 . dim     - The dimension
3912 . simplex - Use simplices, or tensor product cells
3913 - R       - The radius
3914 
3915   Output Parameter:
3916 . dm - The `DM` object
3917 
3918   Level: beginner
3919 
3920 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBallMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3921 @*/
3922 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscReal R, DM *dm)
3923 {
3924   PetscFunctionBegin;
3925   PetscAssertPointer(dm, 5);
3926   PetscCall(DMCreate(comm, dm));
3927   PetscCall(DMSetType(*dm, DMPLEX));
3928   PetscCall(DMPlexCreateSphereMesh_Internal(*dm, dim, simplex, R));
3929   PetscFunctionReturn(PETSC_SUCCESS);
3930 }
3931 
3932 static PetscErrorCode DMPlexCreateBallMesh_Internal(DM dm, PetscInt dim, PetscReal R)
3933 {
3934   DM          sdm, vol;
3935   DMLabel     bdlabel;
3936   const char *prefix;
3937 
3938   PetscFunctionBegin;
3939   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
3940   PetscCall(DMSetType(sdm, DMPLEX));
3941   PetscCall(DMGetOptionsPrefix(dm, &prefix));
3942   PetscCall(DMSetOptionsPrefix(sdm, prefix));
3943   PetscCall(DMAppendOptionsPrefix(sdm, "bd_"));
3944   PetscCall(DMPlexDistributeSetDefault(sdm, PETSC_FALSE));
3945   PetscCall(DMPlexCreateSphereMesh_Internal(sdm, dim - 1, PETSC_TRUE, R));
3946   PetscCall(DMSetFromOptions(sdm));
3947   PetscCall(DMViewFromOptions(sdm, NULL, "-dm_view"));
3948   PetscCall(DMPlexGenerate(sdm, NULL, PETSC_TRUE, &vol));
3949   PetscCall(DMDestroy(&sdm));
3950   PetscCall(DMPlexReplace_Internal(dm, &vol));
3951   PetscCall(DMCreateLabel(dm, "marker"));
3952   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
3953   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
3954   PetscCall(DMPlexLabelComplete(dm, bdlabel));
3955   PetscFunctionReturn(PETSC_SUCCESS);
3956 }
3957 
3958 /*@
3959   DMPlexCreateBallMesh - Creates a simplex mesh on the d-dimensional ball, B^d.
3960 
3961   Collective
3962 
3963   Input Parameters:
3964 + comm - The communicator for the `DM` object
3965 . dim  - The dimension
3966 - R    - The radius
3967 
3968   Output Parameter:
3969 . dm - The `DM` object
3970 
3971   Options Database Key:
3972 . bd_dm_refine - This will refine the surface mesh preserving the sphere geometry
3973 
3974   Level: beginner
3975 
3976 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3977 @*/
3978 PetscErrorCode DMPlexCreateBallMesh(MPI_Comm comm, PetscInt dim, PetscReal R, DM *dm)
3979 {
3980   PetscFunctionBegin;
3981   PetscCall(DMCreate(comm, dm));
3982   PetscCall(DMSetType(*dm, DMPLEX));
3983   PetscCall(DMPlexCreateBallMesh_Internal(*dm, dim, R));
3984   PetscFunctionReturn(PETSC_SUCCESS);
3985 }
3986 
3987 static PetscErrorCode DMPlexCreateReferenceCell_Internal(DM rdm, DMPolytopeType ct)
3988 {
3989   PetscFunctionBegin;
3990   switch (ct) {
3991   case DM_POLYTOPE_POINT: {
3992     PetscInt    numPoints[1]        = {1};
3993     PetscInt    coneSize[1]         = {0};
3994     PetscInt    cones[1]            = {0};
3995     PetscInt    coneOrientations[1] = {0};
3996     PetscScalar vertexCoords[1]     = {0.0};
3997 
3998     PetscCall(DMSetDimension(rdm, 0));
3999     PetscCall(DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4000   } break;
4001   case DM_POLYTOPE_SEGMENT: {
4002     PetscInt    numPoints[2]        = {2, 1};
4003     PetscInt    coneSize[3]         = {2, 0, 0};
4004     PetscInt    cones[2]            = {1, 2};
4005     PetscInt    coneOrientations[2] = {0, 0};
4006     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
4007 
4008     PetscCall(DMSetDimension(rdm, 1));
4009     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4010   } break;
4011   case DM_POLYTOPE_POINT_PRISM_TENSOR: {
4012     PetscInt    numPoints[2]        = {2, 1};
4013     PetscInt    coneSize[3]         = {2, 0, 0};
4014     PetscInt    cones[2]            = {1, 2};
4015     PetscInt    coneOrientations[2] = {0, 0};
4016     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
4017 
4018     PetscCall(DMSetDimension(rdm, 1));
4019     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4020   } break;
4021   case DM_POLYTOPE_TRIANGLE: {
4022     PetscInt    numPoints[2]        = {3, 1};
4023     PetscInt    coneSize[4]         = {3, 0, 0, 0};
4024     PetscInt    cones[3]            = {1, 2, 3};
4025     PetscInt    coneOrientations[3] = {0, 0, 0};
4026     PetscScalar vertexCoords[6]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0};
4027 
4028     PetscCall(DMSetDimension(rdm, 2));
4029     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4030   } break;
4031   case DM_POLYTOPE_QUADRILATERAL: {
4032     PetscInt    numPoints[2]        = {4, 1};
4033     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
4034     PetscInt    cones[4]            = {1, 2, 3, 4};
4035     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
4036     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0};
4037 
4038     PetscCall(DMSetDimension(rdm, 2));
4039     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4040   } break;
4041   case DM_POLYTOPE_SEG_PRISM_TENSOR: {
4042     PetscInt    numPoints[2]        = {4, 1};
4043     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
4044     PetscInt    cones[4]            = {1, 2, 3, 4};
4045     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
4046     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0};
4047 
4048     PetscCall(DMSetDimension(rdm, 2));
4049     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4050   } break;
4051   case DM_POLYTOPE_TETRAHEDRON: {
4052     PetscInt    numPoints[2]        = {4, 1};
4053     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
4054     PetscInt    cones[4]            = {1, 2, 3, 4};
4055     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
4056     PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0};
4057 
4058     PetscCall(DMSetDimension(rdm, 3));
4059     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4060   } break;
4061   case DM_POLYTOPE_HEXAHEDRON: {
4062     PetscInt    numPoints[2]        = {8, 1};
4063     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
4064     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
4065     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
4066     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
4067 
4068     PetscCall(DMSetDimension(rdm, 3));
4069     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4070   } break;
4071   case DM_POLYTOPE_TRI_PRISM: {
4072     PetscInt    numPoints[2]        = {6, 1};
4073     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
4074     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
4075     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
4076     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
4077 
4078     PetscCall(DMSetDimension(rdm, 3));
4079     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4080   } break;
4081   case DM_POLYTOPE_TRI_PRISM_TENSOR: {
4082     PetscInt    numPoints[2]        = {6, 1};
4083     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
4084     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
4085     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
4086     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
4087 
4088     PetscCall(DMSetDimension(rdm, 3));
4089     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4090   } break;
4091   case DM_POLYTOPE_QUAD_PRISM_TENSOR: {
4092     PetscInt    numPoints[2]        = {8, 1};
4093     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
4094     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
4095     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
4096     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
4097 
4098     PetscCall(DMSetDimension(rdm, 3));
4099     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4100   } break;
4101   case DM_POLYTOPE_PYRAMID: {
4102     PetscInt    numPoints[2]        = {5, 1};
4103     PetscInt    coneSize[6]         = {5, 0, 0, 0, 0, 0};
4104     PetscInt    cones[5]            = {1, 2, 3, 4, 5};
4105     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
4106     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, 0.0, 0.0, 1.0};
4107 
4108     PetscCall(DMSetDimension(rdm, 3));
4109     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4110   } break;
4111   default:
4112     SETERRQ(PetscObjectComm((PetscObject)rdm), PETSC_ERR_ARG_WRONG, "Cannot create reference cell for cell type %s", DMPolytopeTypes[ct]);
4113   }
4114   {
4115     PetscInt Nv, v;
4116 
4117     /* Must create the celltype label here so that we do not automatically try to compute the types */
4118     PetscCall(DMCreateLabel(rdm, "celltype"));
4119     PetscCall(DMPlexSetCellType(rdm, 0, ct));
4120     PetscCall(DMPlexGetChart(rdm, NULL, &Nv));
4121     for (v = 1; v < Nv; ++v) PetscCall(DMPlexSetCellType(rdm, v, DM_POLYTOPE_POINT));
4122   }
4123   PetscCall(DMPlexInterpolateInPlace_Internal(rdm));
4124   PetscCall(PetscObjectSetName((PetscObject)rdm, DMPolytopeTypes[ct]));
4125   PetscFunctionReturn(PETSC_SUCCESS);
4126 }
4127 
4128 /*@
4129   DMPlexCreateReferenceCell - Create a `DMPLEX` with the appropriate FEM reference cell
4130 
4131   Collective
4132 
4133   Input Parameters:
4134 + comm - The communicator
4135 - ct   - The cell type of the reference cell
4136 
4137   Output Parameter:
4138 . refdm - The reference cell
4139 
4140   Level: intermediate
4141 
4142 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBoxMesh()`
4143 @*/
4144 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, DMPolytopeType ct, DM *refdm)
4145 {
4146   PetscFunctionBegin;
4147   PetscCall(DMCreate(comm, refdm));
4148   PetscCall(DMSetType(*refdm, DMPLEX));
4149   PetscCall(DMPlexCreateReferenceCell_Internal(*refdm, ct));
4150   PetscFunctionReturn(PETSC_SUCCESS);
4151 }
4152 
4153 static PetscErrorCode DMPlexCreateBoundaryLabel_Private(DM dm, const char name[])
4154 {
4155   DM        plex;
4156   DMLabel   label;
4157   PetscBool hasLabel;
4158 
4159   PetscFunctionBegin;
4160   PetscCall(DMHasLabel(dm, name, &hasLabel));
4161   if (hasLabel) PetscFunctionReturn(PETSC_SUCCESS);
4162   PetscCall(DMCreateLabel(dm, name));
4163   PetscCall(DMGetLabel(dm, name, &label));
4164   PetscCall(DMConvert(dm, DMPLEX, &plex));
4165   PetscCall(DMPlexMarkBoundaryFaces(plex, 1, label));
4166   PetscCall(DMPlexLabelComplete(plex, label));
4167   PetscCall(DMDestroy(&plex));
4168   PetscFunctionReturn(PETSC_SUCCESS);
4169 }
4170 
4171 /*
4172   We use the last coordinate as the radius, the inner radius is lower[dim-1] and the outer radius is upper[dim-1]. Then we map the first coordinate around the circle.
4173 
4174     (x, y) -> (r, theta) = (x[1], (x[0] - lower[0]) * 2\pi/(upper[0] - lower[0]))
4175 */
4176 static void boxToAnnulus(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
4177 {
4178   const PetscReal low = PetscRealPart(constants[0]);
4179   const PetscReal upp = PetscRealPart(constants[1]);
4180   const PetscReal r   = PetscRealPart(u[1]);
4181   const PetscReal th  = 2. * PETSC_PI * (PetscRealPart(u[0]) - low) / (upp - low);
4182 
4183   f0[0] = r * PetscCosReal(th);
4184   f0[1] = r * PetscSinReal(th);
4185 }
4186 
4187 // Insert vertices and their joins, marked by depth
4188 static PetscErrorCode ProcessCohesiveLabel_Vertices(DM dm, DMLabel label, DMLabel vlabel, PetscInt val, PetscInt n, const PetscInt vertices[])
4189 {
4190   PetscFunctionBegin;
4191   PetscCall(DMPlexMarkSubmesh_Interpolated(dm, vlabel, val, PETSC_FALSE, PETSC_FALSE, label, NULL));
4192   PetscFunctionReturn(PETSC_SUCCESS);
4193 }
4194 
4195 // Insert faces and their closures, marked by depth
4196 static PetscErrorCode ProcessCohesiveLabel_Faces(DM dm, DMLabel label, PetscInt n, const PetscInt faces[])
4197 {
4198   PetscFunctionBegin;
4199   for (PetscInt p = 0; p < n; ++p) {
4200     const PetscInt point   = faces[p];
4201     PetscInt      *closure = NULL;
4202     PetscInt       clSize, pdepth;
4203 
4204     PetscCall(DMPlexGetPointDepth(dm, point, &pdepth));
4205     PetscCall(DMLabelSetValue(label, point, pdepth));
4206     PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &clSize, &closure));
4207     for (PetscInt cl = 0; cl < clSize * 2; cl += 2) {
4208       PetscCall(DMPlexGetPointDepth(dm, closure[cl], &pdepth));
4209       PetscCall(DMLabelSetValue(label, closure[cl], pdepth));
4210     }
4211     PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &clSize, &closure));
4212   }
4213   PetscFunctionReturn(PETSC_SUCCESS);
4214 }
4215 
4216 PETSC_EXTERN PetscErrorCode PetscOptionsFindPairPrefix_Private(PetscOptions, const char pre[], const char name[], const char *option[], const char *value[], PetscBool *flg);
4217 
4218 const char *const DMPlexShapes[] = {"box", "box_surface", "ball", "sphere", "cylinder", "schwarz_p", "gyroid", "doublet", "annulus", "hypercubic", "zbox", "unknown", "DMPlexShape", "DM_SHAPE_", NULL};
4219 
4220 static PetscErrorCode DMPlexCreateFromOptions_Internal(PetscOptionItems *PetscOptionsObject, PetscBool *useCoordSpace, DM dm)
4221 {
4222   DMPlexShape    shape   = DM_SHAPE_BOX;
4223   DMPolytopeType cell    = DM_POLYTOPE_TRIANGLE;
4224   PetscInt       dim     = 2;
4225   PetscBool      simplex = PETSC_TRUE, interpolate = PETSC_TRUE, orient = PETSC_FALSE, adjCone = PETSC_FALSE, adjClosure = PETSC_TRUE, refDomain = PETSC_FALSE;
4226   PetscBool      flg, flg2, fflg, strflg, bdfflg, nameflg;
4227   MPI_Comm       comm;
4228   char           filename[PETSC_MAX_PATH_LEN]   = "<unspecified>";
4229   char           bdFilename[PETSC_MAX_PATH_LEN] = "<unspecified>";
4230   char           plexname[PETSC_MAX_PATH_LEN]   = "";
4231   const char    *option;
4232 
4233   PetscFunctionBegin;
4234   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromOptions, dm, 0, 0, 0));
4235   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
4236   /* TODO Turn this into a registration interface */
4237   PetscCall(PetscOptionsString("-dm_plex_filename", "File containing a mesh", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &fflg));
4238   PetscCall(PetscOptionsString("-dm_plex_file_contents", "Contents of a file format in a string", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &strflg));
4239   PetscCall(PetscOptionsString("-dm_plex_boundary_filename", "File containing a mesh boundary", "DMPlexCreateFromFile", bdFilename, bdFilename, sizeof(bdFilename), &bdfflg));
4240   PetscCall(PetscOptionsString("-dm_plex_name", "Name of the mesh in the file", "DMPlexCreateFromFile", plexname, plexname, sizeof(plexname), &nameflg));
4241   PetscCall(PetscOptionsEnum("-dm_plex_cell", "Cell shape", "", DMPolytopeTypes, (PetscEnum)cell, (PetscEnum *)&cell, NULL));
4242   PetscCall(PetscOptionsBool("-dm_plex_reference_cell_domain", "Use a reference cell domain", "", refDomain, &refDomain, NULL));
4243   PetscCall(PetscOptionsEnum("-dm_plex_shape", "Shape for built-in mesh", "", DMPlexShapes, (PetscEnum)shape, (PetscEnum *)&shape, &flg));
4244   PetscCall(PetscOptionsBoundedInt("-dm_plex_dim", "Topological dimension of the mesh", "DMGetDimension", dim, &dim, &flg, 0));
4245   PetscCall(PetscOptionsBool("-dm_plex_simplex", "Mesh cell shape", "", simplex, &simplex, &flg));
4246   PetscCall(PetscOptionsBool("-dm_plex_interpolate", "Flag to create edges and faces automatically", "", interpolate, &interpolate, &flg));
4247   PetscCall(PetscOptionsBool("-dm_plex_orient", "Orient the constructed mesh", "DMPlexOrient", orient, &orient, &flg));
4248   PetscCall(PetscOptionsBool("-dm_plex_adj_cone", "Set adjacency direction", "DMSetBasicAdjacency", adjCone, &adjCone, &flg));
4249   PetscCall(PetscOptionsBool("-dm_plex_adj_closure", "Set adjacency size", "DMSetBasicAdjacency", adjClosure, &adjClosure, &flg2));
4250   if (flg || flg2) PetscCall(DMSetBasicAdjacency(dm, adjCone, adjClosure));
4251 
4252   switch (cell) {
4253   case DM_POLYTOPE_POINT:
4254   case DM_POLYTOPE_SEGMENT:
4255   case DM_POLYTOPE_POINT_PRISM_TENSOR:
4256   case DM_POLYTOPE_TRIANGLE:
4257   case DM_POLYTOPE_QUADRILATERAL:
4258   case DM_POLYTOPE_TETRAHEDRON:
4259   case DM_POLYTOPE_HEXAHEDRON:
4260     *useCoordSpace = PETSC_TRUE;
4261     break;
4262   default:
4263     *useCoordSpace = PETSC_FALSE;
4264     break;
4265   }
4266 
4267   if (fflg) {
4268     DM          dmnew;
4269     const char *name;
4270 
4271     PetscCall(PetscObjectGetName((PetscObject)dm, &name));
4272     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), filename, nameflg ? plexname : name, interpolate, &dmnew));
4273     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4274     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4275   } else if (refDomain) {
4276     PetscCall(DMPlexCreateReferenceCell_Internal(dm, cell));
4277   } else if (bdfflg) {
4278     DM          bdm, dmnew;
4279     const char *name;
4280 
4281     PetscCall(PetscObjectGetName((PetscObject)dm, &name));
4282     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), bdFilename, nameflg ? plexname : name, interpolate, &bdm));
4283     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)bdm, "bd_"));
4284     PetscCall(DMSetFromOptions(bdm));
4285     PetscCall(DMPlexGenerate(bdm, NULL, interpolate, &dmnew));
4286     PetscCall(DMDestroy(&bdm));
4287     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4288     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4289   } else if (strflg) {
4290     DM          dmnew;
4291     PetscViewer viewer;
4292     const char *contents;
4293     char       *strname;
4294     char        tmpdir[PETSC_MAX_PATH_LEN];
4295     char        tmpfilename[PETSC_MAX_PATH_LEN];
4296     char        name[PETSC_MAX_PATH_LEN];
4297     MPI_Comm    comm;
4298     PetscMPIInt rank;
4299 
4300     PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
4301     PetscCallMPI(MPI_Comm_rank(comm, &rank));
4302     PetscCall(PetscStrchr(filename, ':', &strname));
4303     PetscCheck(strname, comm, PETSC_ERR_ARG_WRONG, "File contents must have the form \"ext:string_name\", not %s", filename);
4304     strname[0] = '\0';
4305     ++strname;
4306     PetscCall(PetscDLSym(NULL, strname, (void **)&contents));
4307     PetscCheck(contents, comm, PETSC_ERR_ARG_WRONG, "Could not locate mesh string %s", strname);
4308     PetscCall(PetscGetTmp(comm, tmpdir, PETSC_MAX_PATH_LEN));
4309     PetscCall(PetscStrlcat(tmpdir, "/meshXXXXXX", PETSC_MAX_PATH_LEN));
4310     PetscCall(PetscMkdtemp(tmpdir));
4311     PetscCall(PetscSNPrintf(tmpfilename, PETSC_MAX_PATH_LEN, "%s/mesh.%s", tmpdir, filename));
4312     PetscCall(PetscViewerASCIIOpen(comm, tmpfilename, &viewer));
4313     PetscCall(PetscViewerASCIIPrintf(viewer, "%s\n", contents));
4314     PetscCall(PetscViewerDestroy(&viewer));
4315     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), tmpfilename, plexname, interpolate, &dmnew));
4316     PetscCall(PetscRMTree(tmpdir));
4317     PetscCall(PetscSNPrintf(name, PETSC_MAX_PATH_LEN, "%s Mesh", strname));
4318     PetscCall(PetscObjectSetName((PetscObject)dm, name));
4319     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4320     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4321   } else {
4322     PetscCall(PetscObjectSetName((PetscObject)dm, DMPlexShapes[shape]));
4323     switch (shape) {
4324     case DM_SHAPE_BOX:
4325     case DM_SHAPE_ZBOX:
4326     case DM_SHAPE_ANNULUS: {
4327       PetscInt       faces[3]  = {0, 0, 0};
4328       PetscReal      lower[3]  = {0, 0, 0};
4329       PetscReal      upper[3]  = {1, 1, 1};
4330       DMBoundaryType bdt[3]    = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
4331       PetscBool      isAnnular = shape == DM_SHAPE_ANNULUS ? PETSC_TRUE : PETSC_FALSE;
4332       PetscInt       i, n;
4333 
4334       n = dim;
4335       for (i = 0; i < dim; ++i) faces[i] = (dim == 1 ? 1 : 4 - dim);
4336       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
4337       n = 3;
4338       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
4339       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4340       n = 3;
4341       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
4342       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4343       n = 3;
4344       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
4345       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4346 
4347       PetscCheck(!isAnnular || dim == 2, comm, PETSC_ERR_ARG_OUTOFRANGE, "Only two dimensional annuli have been implemented");
4348       if (isAnnular)
4349         for (i = 0; i < dim - 1; ++i) bdt[i] = DM_BOUNDARY_PERIODIC;
4350 
4351       switch (cell) {
4352       case DM_POLYTOPE_TRI_PRISM_TENSOR:
4353         PetscCall(DMPlexCreateWedgeBoxMesh_Internal(dm, faces, lower, upper, bdt));
4354         if (!interpolate) {
4355           DM udm;
4356 
4357           PetscCall(DMPlexUninterpolate(dm, &udm));
4358           PetscCall(DMPlexReplace_Internal(dm, &udm));
4359         }
4360         break;
4361       default:
4362         PetscCall(DMPlexCreateBoxMesh_Internal(dm, shape, dim, simplex, faces, lower, upper, bdt, interpolate));
4363         break;
4364       }
4365       if (isAnnular) {
4366         DM          cdm;
4367         PetscDS     cds;
4368         PetscScalar bounds[2] = {lower[0], upper[0]};
4369 
4370         // Fix coordinates for annular region
4371         PetscCall(DMSetPeriodicity(dm, NULL, NULL, NULL));
4372         PetscCall(DMSetCellCoordinatesLocal(dm, NULL));
4373         PetscCall(DMSetCellCoordinates(dm, NULL));
4374         PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_TRUE, NULL));
4375         PetscCall(DMGetCoordinateDM(dm, &cdm));
4376         PetscCall(DMGetDS(cdm, &cds));
4377         PetscCall(PetscDSSetConstants(cds, 2, bounds));
4378         PetscCall(DMPlexRemapGeometry(dm, 0.0, boxToAnnulus));
4379       }
4380     } break;
4381     case DM_SHAPE_BOX_SURFACE: {
4382       PetscInt  faces[3] = {0, 0, 0};
4383       PetscReal lower[3] = {0, 0, 0};
4384       PetscReal upper[3] = {1, 1, 1};
4385       PetscInt  i, n;
4386 
4387       n = dim + 1;
4388       for (i = 0; i < dim + 1; ++i) faces[i] = (dim + 1 == 1 ? 1 : 4 - (dim + 1));
4389       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
4390       n = 3;
4391       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
4392       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
4393       n = 3;
4394       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
4395       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
4396       PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(dm, dim + 1, faces, lower, upper, interpolate));
4397     } break;
4398     case DM_SHAPE_SPHERE: {
4399       PetscReal R = 1.0;
4400 
4401       PetscCall(PetscOptionsReal("-dm_plex_sphere_radius", "Radius of the sphere", "", R, &R, &flg));
4402       PetscCall(DMPlexCreateSphereMesh_Internal(dm, dim, simplex, R));
4403     } break;
4404     case DM_SHAPE_BALL: {
4405       PetscReal R = 1.0;
4406 
4407       PetscCall(PetscOptionsReal("-dm_plex_ball_radius", "Radius of the ball", "", R, &R, &flg));
4408       PetscCall(DMPlexCreateBallMesh_Internal(dm, dim, R));
4409     } break;
4410     case DM_SHAPE_CYLINDER: {
4411       DMBoundaryType bdt = DM_BOUNDARY_NONE;
4412       PetscInt       Nw  = 6;
4413       PetscInt       Nr  = 0;
4414 
4415       PetscCall(PetscOptionsEnum("-dm_plex_cylinder_bd", "Boundary type in the z direction", "", DMBoundaryTypes, (PetscEnum)bdt, (PetscEnum *)&bdt, NULL));
4416       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_wedges", "Number of wedges around the cylinder", "", Nw, &Nw, NULL));
4417       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_refine", "Number of refinements before projection", "", Nr, &Nr, NULL));
4418       switch (cell) {
4419       case DM_POLYTOPE_TRI_PRISM_TENSOR:
4420         PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(dm, Nw, interpolate));
4421         break;
4422       default:
4423         PetscCall(DMPlexCreateHexCylinderMesh_Internal(dm, bdt, Nr));
4424         break;
4425       }
4426     } break;
4427     case DM_SHAPE_SCHWARZ_P: // fallthrough
4428     case DM_SHAPE_GYROID: {
4429       PetscInt       extent[3] = {1, 1, 1}, refine = 0, layers = 0, three;
4430       PetscReal      thickness   = 0.;
4431       DMBoundaryType periodic[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
4432       DMPlexTPSType  tps_type    = shape == DM_SHAPE_SCHWARZ_P ? DMPLEX_TPS_SCHWARZ_P : DMPLEX_TPS_GYROID;
4433       PetscBool      tps_distribute;
4434       PetscCall(PetscOptionsIntArray("-dm_plex_tps_extent", "Number of replicas for each of three dimensions", NULL, extent, (three = 3, &three), NULL));
4435       PetscCall(PetscOptionsInt("-dm_plex_tps_refine", "Number of refinements", NULL, refine, &refine, NULL));
4436       PetscCall(PetscOptionsEnumArray("-dm_plex_tps_periodic", "Periodicity in each of three dimensions", NULL, DMBoundaryTypes, (PetscEnum *)periodic, (three = 3, &three), NULL));
4437       PetscCall(PetscOptionsInt("-dm_plex_tps_layers", "Number of layers in volumetric extrusion (or zero to not extrude)", NULL, layers, &layers, NULL));
4438       PetscCall(PetscOptionsReal("-dm_plex_tps_thickness", "Thickness of volumetric extrusion", NULL, thickness, &thickness, NULL));
4439       PetscCall(DMPlexDistributeGetDefault(dm, &tps_distribute));
4440       PetscCall(PetscOptionsBool("-dm_plex_tps_distribute", "Distribute the 2D mesh prior to refinement and extrusion", NULL, tps_distribute, &tps_distribute, NULL));
4441       PetscCall(DMPlexCreateTPSMesh_Internal(dm, tps_type, extent, periodic, tps_distribute, refine, layers, thickness));
4442     } break;
4443     case DM_SHAPE_DOUBLET: {
4444       DM        dmnew;
4445       PetscReal rl = 0.0;
4446 
4447       PetscCall(PetscOptionsReal("-dm_plex_doublet_refinementlimit", "Refinement limit", NULL, rl, &rl, NULL));
4448       PetscCall(DMPlexCreateDoublet(PetscObjectComm((PetscObject)dm), dim, simplex, interpolate, rl, &dmnew));
4449       PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4450       PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4451     } break;
4452     case DM_SHAPE_HYPERCUBIC: {
4453       PetscInt       *edges;
4454       PetscReal      *lower, *upper;
4455       DMBoundaryType *bdt;
4456       PetscInt        n, d;
4457 
4458       *useCoordSpace = PETSC_FALSE;
4459       PetscCall(PetscMalloc4(dim, &edges, dim, &lower, dim, &upper, dim, &bdt));
4460       for (d = 0; d < dim; ++d) {
4461         edges[d] = 1;
4462         lower[d] = 0.;
4463         upper[d] = 1.;
4464         bdt[d]   = DM_BOUNDARY_PERIODIC;
4465       }
4466       n = dim;
4467       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", edges, &n, &flg));
4468       n = dim;
4469       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
4470       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4471       n = dim;
4472       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
4473       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4474       n = dim;
4475       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
4476       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4477       PetscCall(DMPlexCreateHypercubicMesh_Internal(dm, dim, lower, upper, edges, bdt));
4478       PetscCall(PetscFree4(edges, lower, upper, bdt));
4479     } break;
4480     default:
4481       SETERRQ(comm, PETSC_ERR_SUP, "Domain shape %s is unsupported", DMPlexShapes[shape]);
4482     }
4483   }
4484   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
4485   if (!((PetscObject)dm)->name && nameflg) PetscCall(PetscObjectSetName((PetscObject)dm, plexname));
4486   if (orient) PetscCall(DMPlexOrient(dm));
4487   // Allow label creation
4488   PetscCall(PetscOptionsFindPairPrefix_Private(NULL, ((PetscObject)dm)->prefix, "-dm_plex_label_", &option, NULL, &flg));
4489   if (flg) {
4490     DMLabel     label;
4491     PetscInt    points[1024], n = 1024;
4492     char        fulloption[PETSC_MAX_PATH_LEN];
4493     const char *name = &option[14];
4494 
4495     PetscCall(DMCreateLabel(dm, name));
4496     PetscCall(DMGetLabel(dm, name, &label));
4497     fulloption[0] = '-';
4498     fulloption[1] = 0;
4499     PetscCall(PetscStrlcat(fulloption, option, PETSC_MAX_PATH_LEN));
4500     PetscCall(PetscOptionsGetIntArray(NULL, ((PetscObject)dm)->prefix, fulloption, points, &n, NULL));
4501     for (PetscInt p = 0; p < n; ++p) PetscCall(DMLabelSetValue(label, points[p], 1));
4502   }
4503   // Allow cohesive label creation
4504   //   Faces are input, completed, and all points are marked with their depth
4505   PetscCall(PetscOptionsFindPairPrefix_Private(NULL, ((PetscObject)dm)->prefix, "-dm_plex_cohesive_label_", &option, NULL, &flg));
4506   if (flg) {
4507     DMLabel   label;
4508     PetscInt  points[1024], n, pStart, pEnd, Nl = 1;
4509     PetscBool noCreate = PETSC_FALSE;
4510     char      fulloption[PETSC_MAX_PATH_LEN];
4511     char      name[PETSC_MAX_PATH_LEN];
4512     size_t    len;
4513 
4514     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
4515     PetscCall(PetscStrncpy(name, &option[23], PETSC_MAX_PATH_LEN));
4516     PetscCall(PetscStrlen(name, &len));
4517     if (name[len - 1] == '0') Nl = 10;
4518     for (PetscInt l = 0; l < Nl; ++l) {
4519       if (l > 0) name[len - 1] = (char)('0' + l);
4520       fulloption[0] = 0;
4521       PetscCall(PetscStrlcat(fulloption, "-dm_plex_cohesive_label_", 32));
4522       PetscCall(PetscStrlcat(fulloption, name, PETSC_MAX_PATH_LEN - 32));
4523       n = 1024;
4524       PetscCall(PetscOptionsGetIntArray(NULL, ((PetscObject)dm)->prefix, fulloption, points, &n, &flg));
4525       if (!flg) break;
4526       PetscCall(DMHasLabel(dm, name, &noCreate));
4527       if (noCreate) {
4528         DMLabel         inlabel;
4529         IS              pointIS;
4530         const PetscInt *lpoints;
4531         PetscInt        pdep, ln, inval = points[0];
4532         char            newname[PETSC_MAX_PATH_LEN];
4533 
4534         PetscCheck(n == 1, comm, PETSC_ERR_ARG_WRONG, "Must specify a label value with this option");
4535         PetscCall(DMGetLabel(dm, name, &inlabel));
4536         PetscCall(DMLabelGetStratumIS(inlabel, inval, &pointIS));
4537         PetscCall(ISGetLocalSize(pointIS, &ln));
4538         PetscCall(ISGetIndices(pointIS, &lpoints));
4539         PetscCall(DMPlexGetPointDepth(dm, lpoints[0], &pdep));
4540         PetscCall(PetscSNPrintf(newname, PETSC_MAX_PATH_LEN, "%s%" PetscInt_FMT, name, points[0]));
4541         PetscCall(DMCreateLabel(dm, newname));
4542         PetscCall(DMGetLabel(dm, newname, &label));
4543         if (!pdep) PetscCall(ProcessCohesiveLabel_Vertices(dm, label, inlabel, inval, ln, lpoints));
4544         else PetscCall(ProcessCohesiveLabel_Faces(dm, label, ln, lpoints));
4545         PetscCall(ISRestoreIndices(pointIS, &lpoints));
4546         PetscCall(ISDestroy(&pointIS));
4547       } else {
4548         PetscCall(DMCreateLabel(dm, name));
4549         PetscCall(DMGetLabel(dm, name, &label));
4550         if (pStart >= pEnd) n = 0;
4551         PetscCall(ProcessCohesiveLabel_Faces(dm, label, n, points));
4552       }
4553       PetscCall(DMPlexOrientLabel(dm, label));
4554       PetscCall(DMPlexLabelCohesiveComplete(dm, label, NULL, 1, PETSC_FALSE, PETSC_FALSE, NULL));
4555     }
4556   }
4557   PetscCall(DMViewFromOptions(dm, NULL, "-created_dm_view"));
4558   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromOptions, dm, 0, 0, 0));
4559   PetscFunctionReturn(PETSC_SUCCESS);
4560 }
4561 
4562 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
4563 {
4564   DM_Plex  *mesh = (DM_Plex *)dm->data;
4565   PetscBool flg, flg2;
4566   char      bdLabel[PETSC_MAX_PATH_LEN];
4567   char      method[PETSC_MAX_PATH_LEN];
4568 
4569   PetscFunctionBegin;
4570   /* Handle viewing */
4571   PetscCall(PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL));
4572   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fem", "Debug output level for all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL, 0));
4573   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fvm", "Debug output level for all fvm computations", "DMPlexSNESComputeResidualFVM", 0, &mesh->printFVM, NULL, 0));
4574   PetscCall(PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL));
4575   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL, 0));
4576   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_locate", "Debug output level all point location computations", "DMLocatePoints", 0, &mesh->printLocate, NULL, 0));
4577   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_project", "Debug output level all projection computations", "DMPlexProject", 0, &mesh->printProject, NULL, 0));
4578   PetscCall(DMMonitorSetFromOptions(dm, "-dm_plex_monitor_throughput", "Monitor the simulation throughput", "DMPlexMonitorThroughput", DMPlexMonitorThroughput, NULL, &flg));
4579   if (flg) PetscCall(PetscLogDefaultBegin());
4580   /* Labeling */
4581   PetscCall(PetscOptionsString("-dm_plex_boundary_label", "Label to mark the mesh boundary", "", bdLabel, bdLabel, sizeof(bdLabel), &flg));
4582   if (flg) PetscCall(DMPlexCreateBoundaryLabel_Private(dm, bdLabel));
4583   /* Point Location */
4584   PetscCall(PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL));
4585   /* Partitioning and distribution */
4586   PetscCall(PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL));
4587   /* Reordering */
4588   PetscCall(PetscOptionsBool("-dm_reorder_section", "Compute point permutation for local section", "DMReorderSectionSetDefault", PETSC_FALSE, &flg2, &flg));
4589   if (flg) PetscCall(DMReorderSectionSetDefault(dm, flg2 ? DM_REORDER_DEFAULT_TRUE : DM_REORDER_DEFAULT_FALSE));
4590   PetscCall(PetscOptionsString("-dm_reorder_section_type", "Reordering method for local section", "DMReorderSectionSetType", method, method, PETSC_MAX_PATH_LEN, &flg));
4591   if (flg) PetscCall(DMReorderSectionSetType(dm, method));
4592   /* Generation and remeshing */
4593   PetscCall(PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL));
4594   /* Projection behavior */
4595   PetscCall(PetscOptionsBoundedInt("-dm_plex_max_projection_height", "Maximum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL, 0));
4596   PetscCall(PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL));
4597   /* Checking structure */
4598   {
4599     PetscBool all = PETSC_FALSE;
4600 
4601     PetscCall(PetscOptionsBool("-dm_plex_check_all", "Perform all basic checks", "DMPlexCheck", PETSC_FALSE, &all, NULL));
4602     if (all) {
4603       PetscCall(DMPlexCheck(dm));
4604     } else {
4605       PetscCall(PetscOptionsBool("-dm_plex_check_symmetry", "Check that the adjacency information in the mesh is symmetric", "DMPlexCheckSymmetry", PETSC_FALSE, &flg, &flg2));
4606       if (flg && flg2) PetscCall(DMPlexCheckSymmetry(dm));
4607       PetscCall(PetscOptionsBool("-dm_plex_check_skeleton", "Check that each cell has the correct number of vertices (only for homogeneous simplex or tensor meshes)", "DMPlexCheckSkeleton", PETSC_FALSE, &flg, &flg2));
4608       if (flg && flg2) PetscCall(DMPlexCheckSkeleton(dm, 0));
4609       PetscCall(PetscOptionsBool("-dm_plex_check_faces", "Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type", "DMPlexCheckFaces", PETSC_FALSE, &flg, &flg2));
4610       if (flg && flg2) PetscCall(DMPlexCheckFaces(dm, 0));
4611       PetscCall(PetscOptionsBool("-dm_plex_check_geometry", "Check that cells have positive volume", "DMPlexCheckGeometry", PETSC_FALSE, &flg, &flg2));
4612       if (flg && flg2) PetscCall(DMPlexCheckGeometry(dm));
4613       PetscCall(PetscOptionsBool("-dm_plex_check_pointsf", "Check some necessary conditions for PointSF", "DMPlexCheckPointSF", PETSC_FALSE, &flg, &flg2));
4614       if (flg && flg2) PetscCall(DMPlexCheckPointSF(dm, NULL, PETSC_FALSE));
4615       PetscCall(PetscOptionsBool("-dm_plex_check_interface_cones", "Check points on inter-partition interfaces have conforming order of cone points", "DMPlexCheckInterfaceCones", PETSC_FALSE, &flg, &flg2));
4616       if (flg && flg2) PetscCall(DMPlexCheckInterfaceCones(dm));
4617     }
4618     PetscCall(PetscOptionsBool("-dm_plex_check_cell_shape", "Check cell shape", "DMPlexCheckCellShape", PETSC_FALSE, &flg, &flg2));
4619     if (flg && flg2) PetscCall(DMPlexCheckCellShape(dm, PETSC_TRUE, PETSC_DETERMINE));
4620   }
4621   {
4622     PetscReal scale = 1.0;
4623 
4624     PetscCall(PetscOptionsReal("-dm_plex_scale", "Scale factor for mesh coordinates", "DMPlexScale", scale, &scale, &flg));
4625     if (flg) {
4626       Vec coordinates, coordinatesLocal;
4627 
4628       PetscCall(DMGetCoordinates(dm, &coordinates));
4629       PetscCall(DMGetCoordinatesLocal(dm, &coordinatesLocal));
4630       PetscCall(VecScale(coordinates, scale));
4631       PetscCall(VecScale(coordinatesLocal, scale));
4632     }
4633   }
4634   PetscCall(PetscPartitionerSetFromOptions(mesh->partitioner));
4635   PetscFunctionReturn(PETSC_SUCCESS);
4636 }
4637 
4638 PetscErrorCode DMSetFromOptions_Overlap_Plex(DM dm, PetscOptionItems *PetscOptionsObject, PetscInt *overlap)
4639 {
4640   PetscInt  numOvLabels = 16, numOvExLabels = 16;
4641   char     *ovLabelNames[16], *ovExLabelNames[16];
4642   PetscInt  numOvValues = 16, numOvExValues = 16, l;
4643   PetscBool flg;
4644 
4645   PetscFunctionBegin;
4646   PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", *overlap, overlap, NULL, 0));
4647   PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_labels", "List of overlap label names", "DMPlexDistribute", ovLabelNames, &numOvLabels, &flg));
4648   if (!flg) numOvLabels = 0;
4649   if (numOvLabels) {
4650     ((DM_Plex *)dm->data)->numOvLabels = numOvLabels;
4651     for (l = 0; l < numOvLabels; ++l) {
4652       PetscCall(DMGetLabel(dm, ovLabelNames[l], &((DM_Plex *)dm->data)->ovLabels[l]));
4653       PetscCheck(((DM_Plex *)dm->data)->ovLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovLabelNames[l]);
4654       PetscCall(PetscFree(ovLabelNames[l]));
4655     }
4656     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_values", "List of overlap label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovValues, &numOvValues, &flg));
4657     if (!flg) numOvValues = 0;
4658     PetscCheck(numOvLabels == numOvValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvLabels, numOvValues);
4659 
4660     PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_exclude_labels", "List of overlap exclude label names", "DMPlexDistribute", ovExLabelNames, &numOvExLabels, &flg));
4661     if (!flg) numOvExLabels = 0;
4662     ((DM_Plex *)dm->data)->numOvExLabels = numOvExLabels;
4663     for (l = 0; l < numOvExLabels; ++l) {
4664       PetscCall(DMGetLabel(dm, ovExLabelNames[l], &((DM_Plex *)dm->data)->ovExLabels[l]));
4665       PetscCheck(((DM_Plex *)dm->data)->ovExLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovExLabelNames[l]);
4666       PetscCall(PetscFree(ovExLabelNames[l]));
4667     }
4668     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_exclude_values", "List of overlap exclude label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovExValues, &numOvExValues, &flg));
4669     if (!flg) numOvExValues = 0;
4670     PetscCheck(numOvExLabels == numOvExValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of exclude labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvExLabels, numOvExValues);
4671   }
4672   PetscFunctionReturn(PETSC_SUCCESS);
4673 }
4674 
4675 static PetscErrorCode DMSetFromOptions_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
4676 {
4677   PetscFunctionList    ordlist;
4678   char                 oname[256];
4679   char                 sublabelname[PETSC_MAX_PATH_LEN] = "";
4680   DMReorderDefaultFlag reorder;
4681   PetscReal            volume    = -1.0;
4682   PetscInt             prerefine = 0, refine = 0, r, coarsen = 0, overlap = 0, extLayers = 0, dim;
4683   PetscBool            uniformOrig = PETSC_FALSE, created = PETSC_FALSE, uniform = PETSC_TRUE, distribute, saveSF = PETSC_FALSE, interpolate = PETSC_TRUE, coordSpace = PETSC_TRUE, remap = PETSC_TRUE, ghostCells = PETSC_FALSE, isHierarchy, flg;
4684 
4685   PetscFunctionBegin;
4686   PetscOptionsHeadBegin(PetscOptionsObject, "DMPlex Options");
4687   if (dm->cloneOpts) goto non_refine;
4688   /* Handle automatic creation */
4689   PetscCall(DMGetDimension(dm, &dim));
4690   if (dim < 0) {
4691     PetscCall(DMPlexCreateFromOptions_Internal(PetscOptionsObject, &coordSpace, dm));
4692     created = PETSC_TRUE;
4693   }
4694   PetscCall(DMGetDimension(dm, &dim));
4695   /* Handle interpolation before distribution */
4696   PetscCall(PetscOptionsBool("-dm_plex_interpolate_pre", "Flag to interpolate mesh before distribution", "", interpolate, &interpolate, &flg));
4697   if (flg) {
4698     DMPlexInterpolatedFlag interpolated;
4699 
4700     PetscCall(DMPlexIsInterpolated(dm, &interpolated));
4701     if (interpolated == DMPLEX_INTERPOLATED_FULL && !interpolate) {
4702       DM udm;
4703 
4704       PetscCall(DMPlexUninterpolate(dm, &udm));
4705       PetscCall(DMPlexReplace_Internal(dm, &udm));
4706     } else if (interpolated != DMPLEX_INTERPOLATED_FULL && interpolate) {
4707       DM idm;
4708 
4709       PetscCall(DMPlexInterpolate(dm, &idm));
4710       PetscCall(DMPlexReplace_Internal(dm, &idm));
4711     }
4712   }
4713   // Handle submesh selection before distribution
4714   PetscCall(PetscOptionsString("-dm_plex_submesh", "Label to use for submesh selection", "", sublabelname, sublabelname, PETSC_MAX_PATH_LEN, &flg));
4715   if (flg) {
4716     DM              subdm;
4717     DMLabel         label;
4718     IS              valueIS, pointIS;
4719     const PetscInt *values, *points;
4720     PetscBool       markedFaces = PETSC_FALSE;
4721     PetscInt        Nv, value, Np;
4722 
4723     PetscCall(DMGetLabel(dm, sublabelname, &label));
4724     PetscCall(DMLabelGetNumValues(label, &Nv));
4725     PetscCheck(Nv == 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Only a single label value is currently supported for submesh selection, not %" PetscInt_FMT, Nv);
4726     PetscCall(DMLabelGetValueIS(label, &valueIS));
4727     PetscCall(ISGetIndices(valueIS, &values));
4728     value = values[0];
4729     PetscCall(ISRestoreIndices(valueIS, &values));
4730     PetscCall(ISDestroy(&valueIS));
4731     PetscCall(DMLabelGetStratumSize(label, value, &Np));
4732     PetscCall(DMLabelGetStratumIS(label, value, &pointIS));
4733     PetscCall(ISGetIndices(pointIS, &points));
4734     for (PetscInt p = 0; p < Np; ++p) {
4735       PetscInt pdepth;
4736 
4737       PetscCall(DMPlexGetPointDepth(dm, points[p], &pdepth));
4738       if (pdepth) {
4739         markedFaces = PETSC_TRUE;
4740         break;
4741       }
4742     }
4743     PetscCall(ISRestoreIndices(pointIS, &points));
4744     PetscCall(ISDestroy(&pointIS));
4745     PetscCall(DMPlexCreateSubmesh(dm, label, value, markedFaces, &subdm));
4746     PetscCall(DMPlexReplace_Internal(dm, &subdm));
4747     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4748   }
4749   /* Handle DMPlex refinement before distribution */
4750   PetscCall(DMPlexGetRefinementUniform(dm, &uniformOrig));
4751   PetscCall(PetscOptionsBoundedInt("-dm_refine_pre", "The number of refinements before distribution", "DMCreate", prerefine, &prerefine, NULL, 0));
4752   PetscCall(PetscOptionsBool("-dm_refine_remap_pre", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
4753   PetscCall(PetscOptionsBool("-dm_refine_uniform_pre", "Flag for uniform refinement before distribution", "DMCreate", uniform, &uniform, &flg));
4754   if (flg) PetscCall(DMPlexSetRefinementUniform(dm, uniform));
4755   PetscCall(PetscOptionsReal("-dm_refine_volume_limit_pre", "The maximum cell volume after refinement before distribution", "DMCreate", volume, &volume, &flg));
4756   if (flg) {
4757     PetscCall(DMPlexSetRefinementUniform(dm, PETSC_FALSE));
4758     PetscCall(DMPlexSetRefinementLimit(dm, volume));
4759     prerefine = PetscMax(prerefine, 1);
4760   }
4761   if (prerefine) PetscCall(DMLocalizeCoordinates(dm));
4762   for (r = 0; r < prerefine; ++r) {
4763     DM             rdm;
4764     PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4765 
4766     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4767     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
4768     PetscCall(DMPlexReplace_Internal(dm, &rdm));
4769     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4770     if (coordFunc && remap) {
4771       PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4772       ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4773     }
4774   }
4775   PetscCall(DMPlexSetRefinementUniform(dm, uniformOrig));
4776   /* Handle DMPlex extrusion before distribution */
4777   PetscCall(PetscOptionsBoundedInt("-dm_extrude", "The number of layers to extrude", "", extLayers, &extLayers, NULL, 0));
4778   if (extLayers) {
4779     DM edm;
4780 
4781     PetscCall(DMExtrude(dm, extLayers, &edm));
4782     PetscCall(DMPlexReplace_Internal(dm, &edm));
4783     ((DM_Plex *)dm->data)->coordFunc = NULL;
4784     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4785     extLayers = 0;
4786     PetscCall(DMGetDimension(dm, &dim));
4787   }
4788   /* Handle DMPlex reordering before distribution */
4789   PetscCall(DMPlexReorderGetDefault(dm, &reorder));
4790   PetscCall(MatGetOrderingList(&ordlist));
4791   PetscCall(PetscStrncpy(oname, MATORDERINGNATURAL, sizeof(oname)));
4792   PetscCall(PetscOptionsFList("-dm_plex_reorder", "Set mesh reordering type", "DMPlexGetOrdering", ordlist, MATORDERINGNATURAL, oname, sizeof(oname), &flg));
4793   if (reorder == DM_REORDER_DEFAULT_TRUE || flg) {
4794     DM pdm;
4795     IS perm;
4796 
4797     PetscCall(DMPlexGetOrdering(dm, oname, NULL, &perm));
4798     PetscCall(DMPlexPermute(dm, perm, &pdm));
4799     PetscCall(ISDestroy(&perm));
4800     PetscCall(DMPlexReplace_Internal(dm, &pdm));
4801     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4802   }
4803   /* Handle DMPlex distribution */
4804   PetscCall(DMPlexDistributeGetDefault(dm, &distribute));
4805   PetscCall(PetscOptionsBool("-dm_distribute", "Flag to redistribute a mesh among processes", "DMPlexDistribute", distribute, &distribute, NULL));
4806   PetscCall(PetscOptionsBool("-dm_distribute_save_sf", "Flag to save the migration SF", "DMPlexSetMigrationSF", saveSF, &saveSF, NULL));
4807   PetscCall(DMSetFromOptions_Overlap_Plex(dm, PetscOptionsObject, &overlap));
4808   if (distribute) {
4809     DM               pdm = NULL;
4810     PetscPartitioner part;
4811     PetscSF          sfMigration;
4812 
4813     PetscCall(DMPlexGetPartitioner(dm, &part));
4814     PetscCall(PetscPartitionerSetFromOptions(part));
4815     PetscCall(DMPlexDistribute(dm, overlap, &sfMigration, &pdm));
4816     if (pdm) {
4817       // Delete the local section to force the existing one to be rebuilt with the distributed DM
4818       PetscCall(DMSetLocalSection(dm, pdm->localSection));
4819       PetscCall(DMPlexReplace_Internal(dm, &pdm));
4820     }
4821     if (saveSF) PetscCall(DMPlexSetMigrationSF(dm, sfMigration));
4822     PetscCall(PetscSFDestroy(&sfMigration));
4823   }
4824 
4825   {
4826     PetscBool useBoxLabel = PETSC_FALSE;
4827     PetscCall(PetscOptionsBool("-dm_plex_box_label", "Create 'Face Sets' assuming boundary faces align with cartesian directions", "DMCreate", useBoxLabel, &useBoxLabel, NULL));
4828     if (useBoxLabel) {
4829       PetscInt       n      = 3;
4830       DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
4831 
4832       PetscCall(PetscOptionsEnumArray("-dm_plex_box_label_bd", "Boundary type for each dimension when using -dm_plex_box_label", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
4833       PetscCheck(!flg || !(n != dim), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4834       PetscCall(DMPlexSetBoxLabel_Internal(dm, bdt));
4835     }
4836   }
4837   /* Must check CEED options before creating function space for coordinates */
4838   {
4839     PetscBool useCeed = PETSC_FALSE, flg;
4840 
4841     PetscCall(PetscOptionsBool("-dm_plex_use_ceed", "Use LibCEED as the FEM backend", "DMPlexSetUseCeed", useCeed, &useCeed, &flg));
4842     if (flg) PetscCall(DMPlexSetUseCeed(dm, useCeed));
4843   }
4844   /* Create coordinate space */
4845   if (created) {
4846     DM_Plex  *mesh   = (DM_Plex *)dm->data;
4847     PetscInt  degree = 1, deg;
4848     PetscInt  height = 0;
4849     DM        cdm;
4850     PetscBool flg, localize = PETSC_TRUE, sparseLocalize = PETSC_TRUE;
4851 
4852     PetscCall(PetscOptionsBool("-dm_coord_space", "Use an FEM space for coordinates", "", coordSpace, &coordSpace, &flg));
4853     PetscCall(PetscOptionsInt("-dm_coord_petscspace_degree", "FEM degree for coordinate space", "", degree, &degree, NULL));
4854     PetscCall(DMGetCoordinateDegree_Internal(dm, &deg));
4855     if (coordSpace && deg <= 1) PetscCall(DMPlexCreateCoordinateSpace(dm, degree, PETSC_TRUE, mesh->coordFunc));
4856     PetscCall(DMGetCoordinateDM(dm, &cdm));
4857     if (flg && !coordSpace) {
4858       PetscDS      cds;
4859       PetscObject  obj;
4860       PetscClassId id;
4861 
4862       PetscCall(DMGetDS(cdm, &cds));
4863       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
4864       PetscCall(PetscObjectGetClassId(obj, &id));
4865       if (id == PETSCFE_CLASSID) {
4866         PetscContainer dummy;
4867 
4868         PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &dummy));
4869         PetscCall(PetscObjectSetName((PetscObject)dummy, "coordinates"));
4870         PetscCall(DMSetField(cdm, 0, NULL, (PetscObject)dummy));
4871         PetscCall(PetscContainerDestroy(&dummy));
4872         PetscCall(DMClearDS(cdm));
4873       }
4874       mesh->coordFunc = NULL;
4875     }
4876     PetscCall(PetscOptionsBool("-dm_localize", "Localize mesh coordinates", "", localize, &localize, NULL));
4877     PetscCall(PetscOptionsBool("-dm_sparse_localize", "Localize only necessary cells", "DMSetSparseLocalize", sparseLocalize, &sparseLocalize, &flg));
4878     if (flg) PetscCall(DMSetSparseLocalize(dm, sparseLocalize));
4879     PetscCall(PetscOptionsInt("-dm_localize_height", "Localize edges and faces in addition to cells", "", height, &height, &flg));
4880     if (flg) PetscCall(DMPlexSetMaxProjectionHeight(cdm, height));
4881     if (localize) PetscCall(DMLocalizeCoordinates(dm));
4882   }
4883   /* Handle DMPlex refinement */
4884   remap = PETSC_TRUE;
4885   PetscCall(PetscOptionsBoundedInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL, 0));
4886   PetscCall(PetscOptionsBool("-dm_refine_remap", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
4887   PetscCall(PetscOptionsBoundedInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy, 0));
4888   if (refine) PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
4889   if (refine && isHierarchy) {
4890     DM *dms, coarseDM;
4891 
4892     PetscCall(DMGetCoarseDM(dm, &coarseDM));
4893     PetscCall(PetscObjectReference((PetscObject)coarseDM));
4894     PetscCall(PetscMalloc1(refine, &dms));
4895     PetscCall(DMRefineHierarchy(dm, refine, dms));
4896     /* Total hack since we do not pass in a pointer */
4897     PetscCall(DMPlexSwap_Static(dm, dms[refine - 1]));
4898     if (refine == 1) {
4899       PetscCall(DMSetCoarseDM(dm, dms[0]));
4900       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4901     } else {
4902       PetscCall(DMSetCoarseDM(dm, dms[refine - 2]));
4903       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4904       PetscCall(DMSetCoarseDM(dms[0], dms[refine - 1]));
4905       PetscCall(DMPlexSetRegularRefinement(dms[0], PETSC_TRUE));
4906     }
4907     PetscCall(DMSetCoarseDM(dms[refine - 1], coarseDM));
4908     PetscCall(PetscObjectDereference((PetscObject)coarseDM));
4909     /* Free DMs */
4910     for (r = 0; r < refine; ++r) {
4911       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4912       PetscCall(DMDestroy(&dms[r]));
4913     }
4914     PetscCall(PetscFree(dms));
4915   } else {
4916     for (r = 0; r < refine; ++r) {
4917       DM             rdm;
4918       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4919 
4920       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4921       PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
4922       /* Total hack since we do not pass in a pointer */
4923       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4924       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4925       if (coordFunc && remap) {
4926         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4927         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4928       }
4929     }
4930   }
4931   /* Handle DMPlex coarsening */
4932   PetscCall(PetscOptionsBoundedInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL, 0));
4933   PetscCall(PetscOptionsBoundedInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy, 0));
4934   if (coarsen && isHierarchy) {
4935     DM *dms;
4936 
4937     PetscCall(PetscMalloc1(coarsen, &dms));
4938     PetscCall(DMCoarsenHierarchy(dm, coarsen, dms));
4939     /* Free DMs */
4940     for (r = 0; r < coarsen; ++r) {
4941       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4942       PetscCall(DMDestroy(&dms[r]));
4943     }
4944     PetscCall(PetscFree(dms));
4945   } else {
4946     for (r = 0; r < coarsen; ++r) {
4947       DM             cdm;
4948       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4949 
4950       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4951       PetscCall(DMCoarsen(dm, PetscObjectComm((PetscObject)dm), &cdm));
4952       /* Total hack since we do not pass in a pointer */
4953       PetscCall(DMPlexReplace_Internal(dm, &cdm));
4954       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4955       if (coordFunc) {
4956         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4957         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4958       }
4959     }
4960   }
4961   // Handle coordinate remapping
4962   remap = PETSC_FALSE;
4963   PetscCall(PetscOptionsBool("-dm_coord_remap", "Flag to control coordinate remapping", "", remap, &remap, NULL));
4964   if (remap) {
4965     DMPlexCoordMap map     = DM_COORD_MAP_NONE;
4966     PetscPointFunc mapFunc = NULL;
4967     PetscScalar    params[16];
4968     PetscInt       Np = PETSC_STATIC_ARRAY_LENGTH(params), cdim;
4969     MPI_Comm       comm;
4970 
4971     PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
4972     PetscCall(DMGetCoordinateDim(dm, &cdim));
4973     PetscCall(PetscOptionsScalarArray("-dm_coord_map_params", "Parameters for the coordinate remapping", "", params, &Np, &flg));
4974     if (!flg) Np = 0;
4975     // TODO Allow user to pass a map function by name
4976     PetscCall(PetscOptionsEnum("-dm_coord_map", "Coordinate mapping for built-in mesh", "", DMPlexCoordMaps, (PetscEnum)map, (PetscEnum *)&map, &flg));
4977     if (flg) {
4978       switch (map) {
4979       case DM_COORD_MAP_NONE:
4980         mapFunc = coordMap_identity;
4981         break;
4982       case DM_COORD_MAP_SHEAR:
4983         mapFunc = coordMap_shear;
4984         if (!Np) {
4985           Np        = cdim + 1;
4986           params[0] = 0;
4987           for (PetscInt d = 1; d <= cdim; ++d) params[d] = 1.0;
4988         }
4989         PetscCheck(Np == cdim + 1, comm, PETSC_ERR_ARG_WRONG, "The shear coordinate map must have cdim + 1 = %" PetscInt_FMT " parameters, not %" PetscInt_FMT, cdim + 1, Np);
4990         break;
4991       case DM_COORD_MAP_FLARE:
4992         mapFunc = coordMap_flare;
4993         if (!Np) {
4994           Np        = cdim + 1;
4995           params[0] = 0;
4996           for (PetscInt d = 1; d <= cdim; ++d) params[d] = 1.0;
4997         }
4998         PetscCheck(Np == cdim + 1, comm, PETSC_ERR_ARG_WRONG, "The flare coordinate map must have cdim + 1 = %" PetscInt_FMT " parameters, not %" PetscInt_FMT, cdim + 1, Np);
4999         break;
5000       case DM_COORD_MAP_ANNULUS:
5001         mapFunc = coordMap_annulus;
5002         if (!Np) {
5003           Np        = 2;
5004           params[0] = 1.;
5005           params[1] = 2.;
5006         }
5007         PetscCheck(Np == 2, comm, PETSC_ERR_ARG_WRONG, "The annulus coordinate map must have 2 parameters, not %" PetscInt_FMT, Np);
5008         break;
5009       case DM_COORD_MAP_SHELL:
5010         mapFunc = coordMap_shell;
5011         if (!Np) {
5012           Np        = 2;
5013           params[0] = 1.;
5014           params[1] = 2.;
5015         }
5016         PetscCheck(Np == 2, comm, PETSC_ERR_ARG_WRONG, "The spherical shell coordinate map must have 2 parameters, not %" PetscInt_FMT, Np);
5017         break;
5018       default:
5019         mapFunc = coordMap_identity;
5020       }
5021     }
5022     if (Np) {
5023       DM      cdm;
5024       PetscDS cds;
5025 
5026       PetscCall(DMGetCoordinateDM(dm, &cdm));
5027       PetscCall(DMGetDS(cdm, &cds));
5028       PetscCall(PetscDSSetConstants(cds, Np, params));
5029     }
5030     PetscCall(DMPlexRemapGeometry(dm, 0.0, mapFunc));
5031   }
5032   /* Handle ghost cells */
5033   PetscCall(PetscOptionsBool("-dm_plex_create_fv_ghost_cells", "Flag to create finite volume ghost cells on the boundary", "DMCreate", ghostCells, &ghostCells, NULL));
5034   if (ghostCells) {
5035     DM   gdm;
5036     char lname[PETSC_MAX_PATH_LEN];
5037 
5038     lname[0] = '\0';
5039     PetscCall(PetscOptionsString("-dm_plex_fv_ghost_cells_label", "Label name for ghost cells boundary", "DMCreate", lname, lname, sizeof(lname), &flg));
5040     PetscCall(DMPlexConstructGhostCells(dm, flg ? lname : NULL, NULL, &gdm));
5041     PetscCall(DMPlexReplace_Internal(dm, &gdm));
5042   }
5043   /* Handle 1D order */
5044   if (reorder != DM_REORDER_DEFAULT_FALSE && dim == 1) {
5045     DM           cdm, rdm;
5046     PetscDS      cds;
5047     PetscObject  obj;
5048     PetscClassId id = PETSC_OBJECT_CLASSID;
5049     IS           perm;
5050     PetscInt     Nf;
5051     PetscBool    distributed;
5052 
5053     PetscCall(DMPlexIsDistributed(dm, &distributed));
5054     PetscCall(DMGetCoordinateDM(dm, &cdm));
5055     PetscCall(DMGetDS(cdm, &cds));
5056     PetscCall(PetscDSGetNumFields(cds, &Nf));
5057     if (Nf) {
5058       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
5059       PetscCall(PetscObjectGetClassId(obj, &id));
5060     }
5061     if (!distributed && id != PETSCFE_CLASSID) {
5062       PetscCall(DMPlexGetOrdering1D(dm, &perm));
5063       PetscCall(DMPlexPermute(dm, perm, &rdm));
5064       PetscCall(DMPlexReplace_Internal(dm, &rdm));
5065       PetscCall(ISDestroy(&perm));
5066     }
5067   }
5068 /* Handle */
5069 non_refine:
5070   PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5071   char    *phases[16];
5072   PetscInt Nphases = 16;
5073   PetscCall(PetscOptionsStringArray("-dm_plex_option_phases", "Option phase prefixes", "DMSetFromOptions", phases, &Nphases, &flg));
5074   PetscOptionsHeadEnd();
5075 
5076   // Phases
5077   if (flg) {
5078     const char *oldPrefix;
5079 
5080     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &oldPrefix));
5081     for (PetscInt ph = 0; ph < Nphases; ++ph) {
5082       PetscCall(PetscObjectAppendOptionsPrefix((PetscObject)dm, phases[ph]));
5083       PetscCall(PetscInfo(dm, "Options phase %s for DM %s\n", phases[ph], dm->hdr.name));
5084       PetscCall(DMSetFromOptions(dm));
5085       PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, oldPrefix));
5086       PetscCall(PetscFree(phases[ph]));
5087     }
5088   }
5089   PetscFunctionReturn(PETSC_SUCCESS);
5090 }
5091 
5092 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm, Vec *vec)
5093 {
5094   PetscFunctionBegin;
5095   PetscCall(DMCreateGlobalVector_Section_Private(dm, vec));
5096   /* PetscCall(VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM)); */
5097   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex));
5098   PetscCall(VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void))VecView_Plex_Native));
5099   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex));
5100   PetscCall(VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void))VecLoad_Plex_Native));
5101   PetscFunctionReturn(PETSC_SUCCESS);
5102 }
5103 
5104 static PetscErrorCode DMCreateLocalVector_Plex(DM dm, Vec *vec)
5105 {
5106   PetscFunctionBegin;
5107   PetscCall(DMCreateLocalVector_Section_Private(dm, vec));
5108   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex_Local));
5109   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex_Local));
5110   PetscFunctionReturn(PETSC_SUCCESS);
5111 }
5112 
5113 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
5114 {
5115   PetscInt depth, d;
5116 
5117   PetscFunctionBegin;
5118   PetscCall(DMPlexGetDepth(dm, &depth));
5119   if (depth == 1) {
5120     PetscCall(DMGetDimension(dm, &d));
5121     if (dim == 0) PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
5122     else if (dim == d) PetscCall(DMPlexGetDepthStratum(dm, 1, pStart, pEnd));
5123     else {
5124       *pStart = 0;
5125       *pEnd   = 0;
5126     }
5127   } else {
5128     PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
5129   }
5130   PetscFunctionReturn(PETSC_SUCCESS);
5131 }
5132 
5133 static PetscErrorCode DMGetNeighbors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
5134 {
5135   PetscSF            sf;
5136   PetscMPIInt        niranks, njranks;
5137   PetscInt           n;
5138   const PetscMPIInt *iranks, *jranks;
5139   DM_Plex           *data = (DM_Plex *)dm->data;
5140 
5141   PetscFunctionBegin;
5142   PetscCall(DMGetPointSF(dm, &sf));
5143   if (!data->neighbors) {
5144     PetscCall(PetscSFSetUp(sf));
5145     PetscCall(PetscSFGetRootRanks(sf, &njranks, &jranks, NULL, NULL, NULL));
5146     PetscCall(PetscSFGetLeafRanks(sf, &niranks, &iranks, NULL, NULL));
5147     PetscCall(PetscMalloc1(njranks + niranks + 1, &data->neighbors));
5148     PetscCall(PetscArraycpy(data->neighbors + 1, jranks, njranks));
5149     PetscCall(PetscArraycpy(data->neighbors + njranks + 1, iranks, niranks));
5150     n = njranks + niranks;
5151     PetscCall(PetscSortRemoveDupsMPIInt(&n, data->neighbors + 1));
5152     /* The following cast should never fail: can't have more neighbors than PETSC_MPI_INT_MAX */
5153     PetscCall(PetscMPIIntCast(n, data->neighbors));
5154   }
5155   if (nranks) *nranks = data->neighbors[0];
5156   if (ranks) {
5157     if (data->neighbors[0]) *ranks = data->neighbors + 1;
5158     else *ranks = NULL;
5159   }
5160   PetscFunctionReturn(PETSC_SUCCESS);
5161 }
5162 
5163 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM, DM, Mat, Vec, Vec);
5164 
5165 static PetscErrorCode DMInitialize_Plex(DM dm)
5166 {
5167   PetscFunctionBegin;
5168   dm->ops->view                      = DMView_Plex;
5169   dm->ops->load                      = DMLoad_Plex;
5170   dm->ops->setfromoptions            = DMSetFromOptions_Plex;
5171   dm->ops->clone                     = DMClone_Plex;
5172   dm->ops->setup                     = DMSetUp_Plex;
5173   dm->ops->createlocalsection        = DMCreateLocalSection_Plex;
5174   dm->ops->createsectionpermutation  = DMCreateSectionPermutation_Plex;
5175   dm->ops->createdefaultconstraints  = DMCreateDefaultConstraints_Plex;
5176   dm->ops->createglobalvector        = DMCreateGlobalVector_Plex;
5177   dm->ops->createlocalvector         = DMCreateLocalVector_Plex;
5178   dm->ops->getlocaltoglobalmapping   = NULL;
5179   dm->ops->createfieldis             = NULL;
5180   dm->ops->createcoordinatedm        = DMCreateCoordinateDM_Plex;
5181   dm->ops->createcoordinatefield     = DMCreateCoordinateField_Plex;
5182   dm->ops->getcoloring               = NULL;
5183   dm->ops->creatematrix              = DMCreateMatrix_Plex;
5184   dm->ops->createinterpolation       = DMCreateInterpolation_Plex;
5185   dm->ops->createmassmatrix          = DMCreateMassMatrix_Plex;
5186   dm->ops->createmassmatrixlumped    = DMCreateMassMatrixLumped_Plex;
5187   dm->ops->createinjection           = DMCreateInjection_Plex;
5188   dm->ops->refine                    = DMRefine_Plex;
5189   dm->ops->coarsen                   = DMCoarsen_Plex;
5190   dm->ops->refinehierarchy           = DMRefineHierarchy_Plex;
5191   dm->ops->coarsenhierarchy          = DMCoarsenHierarchy_Plex;
5192   dm->ops->extrude                   = DMExtrude_Plex;
5193   dm->ops->globaltolocalbegin        = NULL;
5194   dm->ops->globaltolocalend          = NULL;
5195   dm->ops->localtoglobalbegin        = NULL;
5196   dm->ops->localtoglobalend          = NULL;
5197   dm->ops->destroy                   = DMDestroy_Plex;
5198   dm->ops->createsubdm               = DMCreateSubDM_Plex;
5199   dm->ops->createsuperdm             = DMCreateSuperDM_Plex;
5200   dm->ops->getdimpoints              = DMGetDimPoints_Plex;
5201   dm->ops->locatepoints              = DMLocatePoints_Plex;
5202   dm->ops->projectfunctionlocal      = DMProjectFunctionLocal_Plex;
5203   dm->ops->projectfunctionlabellocal = DMProjectFunctionLabelLocal_Plex;
5204   dm->ops->projectfieldlocal         = DMProjectFieldLocal_Plex;
5205   dm->ops->projectfieldlabellocal    = DMProjectFieldLabelLocal_Plex;
5206   dm->ops->projectbdfieldlabellocal  = DMProjectBdFieldLabelLocal_Plex;
5207   dm->ops->computel2diff             = DMComputeL2Diff_Plex;
5208   dm->ops->computel2gradientdiff     = DMComputeL2GradientDiff_Plex;
5209   dm->ops->computel2fielddiff        = DMComputeL2FieldDiff_Plex;
5210   dm->ops->getneighbors              = DMGetNeighbors_Plex;
5211   dm->ops->getlocalboundingbox       = DMGetLocalBoundingBox_Coordinates;
5212   dm->ops->createdomaindecomposition = DMCreateDomainDecomposition_Plex;
5213   dm->ops->createddscatters          = DMCreateDomainDecompositionScatters_Plex;
5214   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertBoundaryValues_C", DMPlexInsertBoundaryValues_Plex));
5215   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertTimeDerivativeBoundaryValues_C", DMPlexInsertTimeDerivativeBoundaryValues_Plex));
5216   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMSetUpGLVisViewer_C", DMSetUpGLVisViewer_Plex));
5217   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMCreateNeumannOverlap_C", DMCreateNeumannOverlap_Plex));
5218   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeGetDefault_C", DMPlexDistributeGetDefault_Plex));
5219   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeSetDefault_C", DMPlexDistributeSetDefault_Plex));
5220   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderGetDefault_C", DMPlexReorderGetDefault_Plex));
5221   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderSetDefault_C", DMPlexReorderSetDefault_Plex));
5222   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionGetDefault_C", DMReorderSectionGetDefault_Plex));
5223   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionSetDefault_C", DMReorderSectionSetDefault_Plex));
5224   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionGetType_C", DMReorderSectionGetType_Plex));
5225   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionSetType_C", DMReorderSectionSetType_Plex));
5226   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMInterpolateSolution_C", DMInterpolateSolution_Plex));
5227   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
5228   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetOverlap_C", DMPlexSetOverlap_Plex));
5229   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetUseCeed_C", DMPlexGetUseCeed_Plex));
5230   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetUseCeed_C", DMPlexSetUseCeed_Plex));
5231   PetscFunctionReturn(PETSC_SUCCESS);
5232 }
5233 
5234 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
5235 {
5236   DM_Plex       *mesh = (DM_Plex *)dm->data;
5237   const PetscSF *face_sfs;
5238   PetscInt       num_face_sfs;
5239 
5240   PetscFunctionBegin;
5241   mesh->refct++;
5242   (*newdm)->data = mesh;
5243   PetscCall(DMPlexGetIsoperiodicFaceSF(dm, &num_face_sfs, &face_sfs));
5244   PetscCall(DMPlexSetIsoperiodicFaceSF(*newdm, num_face_sfs, (PetscSF *)face_sfs));
5245   PetscCall(PetscObjectChangeTypeName((PetscObject)*newdm, DMPLEX));
5246   PetscCall(DMInitialize_Plex(*newdm));
5247   PetscFunctionReturn(PETSC_SUCCESS);
5248 }
5249 
5250 /*MC
5251   DMPLEX = "plex" - A `DM` object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
5252                     In the local representation, `Vec`s contain all unknowns in the interior and shared boundary. This is
5253                     specified by a PetscSection object. Ownership in the global representation is determined by
5254                     ownership of the underlying `DMPLEX` points. This is specified by another `PetscSection` object.
5255 
5256   Options Database Keys:
5257 + -dm_refine_pre                     - Refine mesh before distribution
5258 + -dm_refine_uniform_pre             - Choose uniform or generator-based refinement
5259 + -dm_refine_volume_limit_pre        - Cell volume limit after pre-refinement using generator
5260 . -dm_distribute                     - Distribute mesh across processes
5261 . -dm_distribute_overlap             - Number of cells to overlap for distribution
5262 . -dm_refine                         - Refine mesh after distribution
5263 . -dm_localize <bool>                - Whether to localize coordinates for periodic meshes
5264 . -dm_sparse_localize <bool>         - Whether to only localize cells on the periodic boundary
5265 . -dm_plex_hash_location             - Use grid hashing for point location
5266 . -dm_plex_hash_box_faces <n,m,p>    - The number of divisions in each direction of the grid hash
5267 . -dm_plex_partition_balance         - Attempt to evenly divide points on partition boundary between processes
5268 . -dm_plex_remesh_bd                 - Allow changes to the boundary on remeshing
5269 . -dm_plex_max_projection_height     - Maximum mesh point height used to project locally
5270 . -dm_plex_regular_refinement        - Use special nested projection algorithm for regular refinement
5271 . -dm_plex_reorder_section           - Use specialized blocking if available
5272 . -dm_plex_check_all                 - Perform all checks below
5273 . -dm_plex_check_symmetry            - Check that the adjacency information in the mesh is symmetric
5274 . -dm_plex_check_skeleton <celltype> - Check that each cell has the correct number of vertices
5275 . -dm_plex_check_faces <celltype>    - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
5276 . -dm_plex_check_geometry            - Check that cells have positive volume
5277 . -dm_view :mesh.tex:ascii_latex     - View the mesh in LaTeX/TikZ
5278 . -dm_plex_view_scale <num>          - Scale the TikZ
5279 . -dm_plex_print_fem <num>           - View FEM assembly information, such as element vectors and matrices
5280 - -dm_plex_print_fvm <num>           - View FVM assembly information, such as flux updates
5281 
5282   Level: intermediate
5283 
5284 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`, `PetscSection`
5285 M*/
5286 
5287 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
5288 {
5289   DM_Plex *mesh;
5290   PetscInt unit;
5291 
5292   PetscFunctionBegin;
5293   PetscCall(PetscCitationsRegister(PlexCitation, &Plexcite));
5294   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5295   PetscCall(PetscNew(&mesh));
5296   dm->reorderSection = DM_REORDER_DEFAULT_NOTSET;
5297   dm->data           = mesh;
5298 
5299   mesh->refct = 1;
5300   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection));
5301   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection));
5302   mesh->refinementUniform      = PETSC_TRUE;
5303   mesh->refinementLimit        = -1.0;
5304   mesh->distDefault            = PETSC_TRUE;
5305   mesh->reorderDefault         = DM_REORDER_DEFAULT_NOTSET;
5306   mesh->distributionName       = NULL;
5307   mesh->interpolated           = DMPLEX_INTERPOLATED_INVALID;
5308   mesh->interpolatedCollective = DMPLEX_INTERPOLATED_INVALID;
5309 
5310   PetscCall(PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner));
5311   mesh->remeshBd = PETSC_FALSE;
5312 
5313   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
5314 
5315   mesh->depthState    = -1;
5316   mesh->celltypeState = -1;
5317   mesh->printTol      = 1.0e-10;
5318   mesh->nonempty_comm = MPI_COMM_SELF;
5319 
5320   PetscCall(DMInitialize_Plex(dm));
5321   PetscFunctionReturn(PETSC_SUCCESS);
5322 }
5323 
5324 /*@
5325   DMPlexCreate - Creates a `DMPLEX` object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
5326 
5327   Collective
5328 
5329   Input Parameter:
5330 . comm - The communicator for the `DMPLEX` object
5331 
5332   Output Parameter:
5333 . mesh - The `DMPLEX` object
5334 
5335   Level: beginner
5336 
5337 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMType`, `DMCreate()`, `DMSetType()`
5338 @*/
5339 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
5340 {
5341   PetscFunctionBegin;
5342   PetscAssertPointer(mesh, 2);
5343   PetscCall(DMCreate(comm, mesh));
5344   PetscCall(DMSetType(*mesh, DMPLEX));
5345   PetscFunctionReturn(PETSC_SUCCESS);
5346 }
5347 
5348 /*@C
5349   DMPlexBuildFromCellListParallel - Build distributed `DMPLEX` topology from a list of vertices for each cell (common mesh generator output) where all cells have the same celltype
5350 
5351   Collective; No Fortran Support
5352 
5353   Input Parameters:
5354 + dm          - The `DM`
5355 . numCells    - The number of cells owned by this process
5356 . numVertices - The number of vertices to be owned by this process, or `PETSC_DECIDE`
5357 . NVertices   - The global number of vertices, or `PETSC_DETERMINE`
5358 . numCorners  - The number of vertices for each cell
5359 - cells       - An array of numCells*numCorners numbers, the global vertex numbers for each cell
5360 
5361   Output Parameters:
5362 + vertexSF         - (Optional) `PetscSF` describing complete vertex ownership
5363 - verticesAdjSaved - (Optional) vertex adjacency array
5364 
5365   Level: advanced
5366 
5367   Notes:
5368   Two triangles sharing a face
5369 .vb
5370 
5371         2
5372       / | \
5373      /  |  \
5374     /   |   \
5375    0  0 | 1  3
5376     \   |   /
5377      \  |  /
5378       \ | /
5379         1
5380 .ve
5381   would have input
5382 .vb
5383   numCells = 2, numVertices = 4
5384   cells = [0 1 2  1 3 2]
5385 .ve
5386   which would result in the `DMPLEX`
5387 .vb
5388 
5389         4
5390       / | \
5391      /  |  \
5392     /   |   \
5393    2  0 | 1  5
5394     \   |   /
5395      \  |  /
5396       \ | /
5397         3
5398 .ve
5399 
5400   Vertices are implicitly numbered consecutively 0,...,NVertices.
5401   Each rank owns a chunk of numVertices consecutive vertices.
5402   If numVertices is `PETSC_DECIDE`, PETSc will distribute them as evenly as possible using PetscLayout.
5403   If NVertices is `PETSC_DETERMINE` and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
5404   If only NVertices is `PETSC_DETERMINE`, it is computed as the sum of numVertices over all ranks.
5405 
5406   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
5407 
5408 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildCoordinatesFromCellListParallel()`,
5409           `PetscSF`
5410 @*/
5411 PetscErrorCode DMPlexBuildFromCellListParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, const PetscInt cells[], PetscSF *vertexSF, PetscInt **verticesAdjSaved)
5412 {
5413   PetscSF     sfPoint;
5414   PetscLayout layout;
5415   PetscInt    numVerticesAdj, *verticesAdj, *cones, c, p;
5416 
5417   PetscFunctionBegin;
5418   PetscValidLogicalCollectiveInt(dm, NVertices, 4);
5419   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5420   /* Get/check global number of vertices */
5421   {
5422     PetscInt       NVerticesInCells, i;
5423     const PetscInt len = numCells * numCorners;
5424 
5425     /* NVerticesInCells = max(cells) + 1 */
5426     NVerticesInCells = PETSC_INT_MIN;
5427     for (i = 0; i < len; i++)
5428       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
5429     ++NVerticesInCells;
5430     PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
5431 
5432     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
5433     else
5434       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
5435   }
5436   /* Count locally unique vertices */
5437   {
5438     PetscHSetI vhash;
5439     PetscInt   off = 0;
5440 
5441     PetscCall(PetscHSetICreate(&vhash));
5442     for (c = 0; c < numCells; ++c) {
5443       for (p = 0; p < numCorners; ++p) PetscCall(PetscHSetIAdd(vhash, cells[c * numCorners + p]));
5444     }
5445     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
5446     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
5447     else verticesAdj = *verticesAdjSaved;
5448     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
5449     PetscCall(PetscHSetIDestroy(&vhash));
5450     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
5451   }
5452   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
5453   /* Create cones */
5454   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
5455   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
5456   PetscCall(DMSetUp(dm));
5457   PetscCall(DMPlexGetCones(dm, &cones));
5458   for (c = 0; c < numCells; ++c) {
5459     for (p = 0; p < numCorners; ++p) {
5460       const PetscInt gv = cells[c * numCorners + p];
5461       PetscInt       lv;
5462 
5463       /* Positions within verticesAdj form 0-based local vertex numbering;
5464          we need to shift it by numCells to get correct DAG points (cells go first) */
5465       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
5466       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
5467       cones[c * numCorners + p] = lv + numCells;
5468     }
5469   }
5470   /* Build point sf */
5471   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
5472   PetscCall(PetscLayoutSetSize(layout, NVertices));
5473   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
5474   PetscCall(PetscLayoutSetBlockSize(layout, 1));
5475   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
5476   PetscCall(PetscLayoutDestroy(&layout));
5477   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
5478   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
5479   if (dm->sf) {
5480     const char *prefix;
5481 
5482     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
5483     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
5484   }
5485   PetscCall(DMSetPointSF(dm, sfPoint));
5486   PetscCall(PetscSFDestroy(&sfPoint));
5487   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)*vertexSF, "Vertex Ownership SF"));
5488   /* Fill in the rest of the topology structure */
5489   PetscCall(DMPlexSymmetrize(dm));
5490   PetscCall(DMPlexStratify(dm));
5491   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5492   PetscFunctionReturn(PETSC_SUCCESS);
5493 }
5494 
5495 /*@C
5496   DMPlexBuildFromCellSectionParallel - Build distributed `DMPLEX` topology from a list of vertices for each cell (common mesh generator output) allowing multiple celltypes
5497 
5498   Collective; No Fortran Support
5499 
5500   Input Parameters:
5501 + dm          - The `DM`
5502 . numCells    - The number of cells owned by this process
5503 . numVertices - The number of vertices to be owned by this process, or `PETSC_DECIDE`
5504 . NVertices   - The global number of vertices, or `PETSC_DETERMINE`
5505 . cellSection - The `PetscSection` giving the number of vertices for each cell (layout of cells)
5506 - cells       - An array of the global vertex numbers for each cell
5507 
5508   Output Parameters:
5509 + vertexSF         - (Optional) `PetscSF` describing complete vertex ownership
5510 - verticesAdjSaved - (Optional) vertex adjacency array
5511 
5512   Level: advanced
5513 
5514   Notes:
5515   A triangle and quadrilateral sharing a face
5516 .vb
5517         2----------3
5518       / |          |
5519      /  |          |
5520     /   |          |
5521    0  0 |     1    |
5522     \   |          |
5523      \  |          |
5524       \ |          |
5525         1----------4
5526 .ve
5527   would have input
5528 .vb
5529   numCells = 2, numVertices = 5
5530   cells = [0 1 2  1 4 3 2]
5531 .ve
5532   which would result in the `DMPLEX`
5533 .vb
5534         4----------5
5535       / |          |
5536      /  |          |
5537     /   |          |
5538    2  0 |     1    |
5539     \   |          |
5540      \  |          |
5541       \ |          |
5542         3----------6
5543 .ve
5544 
5545   Vertices are implicitly numbered consecutively 0,...,NVertices.
5546   Each rank owns a chunk of numVertices consecutive vertices.
5547   If numVertices is `PETSC_DECIDE`, PETSc will distribute them as evenly as possible using PetscLayout.
5548   If NVertices is `PETSC_DETERMINE` and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
5549   If only NVertices is `PETSC_DETERMINE`, it is computed as the sum of numVertices over all ranks.
5550 
5551   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
5552 
5553 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellListParallel()`, `DMPlexCreateFromCellSectionParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`,
5554           `PetscSF`
5555 @*/
5556 PetscErrorCode DMPlexBuildFromCellSectionParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscSection cellSection, const PetscInt cells[], PetscSF *vertexSF, PetscInt **verticesAdjSaved)
5557 {
5558   PetscSF     sfPoint;
5559   PetscLayout layout;
5560   PetscInt    numVerticesAdj, *verticesAdj, *cones, cStart, cEnd, len;
5561 
5562   PetscFunctionBegin;
5563   PetscValidLogicalCollectiveInt(dm, NVertices, 4);
5564   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5565   PetscCall(PetscSectionGetChart(cellSection, &cStart, &cEnd));
5566   PetscCall(PetscSectionGetStorageSize(cellSection, &len));
5567   PetscCheck(cStart == 0 && cEnd == numCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Section chart [%" PetscInt_FMT ", %" PetscInt_FMT ") should be [0, %" PetscInt_FMT ")", cStart, cEnd, numCells);
5568   /* Get/check global number of vertices */
5569   {
5570     PetscInt NVerticesInCells;
5571 
5572     /* NVerticesInCells = max(cells) + 1 */
5573     NVerticesInCells = PETSC_MIN_INT;
5574     for (PetscInt i = 0; i < len; i++)
5575       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
5576     ++NVerticesInCells;
5577     PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
5578 
5579     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
5580     else
5581       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
5582   }
5583   /* Count locally unique vertices */
5584   {
5585     PetscHSetI vhash;
5586     PetscInt   off = 0;
5587 
5588     PetscCall(PetscHSetICreate(&vhash));
5589     for (PetscInt i = 0; i < len; i++) PetscCall(PetscHSetIAdd(vhash, cells[i]));
5590     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
5591     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
5592     else verticesAdj = *verticesAdjSaved;
5593     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
5594     PetscCall(PetscHSetIDestroy(&vhash));
5595     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
5596   }
5597   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
5598   /* Create cones */
5599   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
5600   for (PetscInt c = 0; c < numCells; ++c) {
5601     PetscInt dof;
5602 
5603     PetscCall(PetscSectionGetDof(cellSection, c, &dof));
5604     PetscCall(DMPlexSetConeSize(dm, c, dof));
5605   }
5606   PetscCall(DMSetUp(dm));
5607   PetscCall(DMPlexGetCones(dm, &cones));
5608   for (PetscInt c = 0; c < numCells; ++c) {
5609     PetscInt dof, off;
5610 
5611     PetscCall(PetscSectionGetDof(cellSection, c, &dof));
5612     PetscCall(PetscSectionGetOffset(cellSection, c, &off));
5613     for (PetscInt p = off; p < off + dof; ++p) {
5614       const PetscInt gv = cells[p];
5615       PetscInt       lv;
5616 
5617       /* Positions within verticesAdj form 0-based local vertex numbering;
5618          we need to shift it by numCells to get correct DAG points (cells go first) */
5619       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
5620       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
5621       cones[p] = lv + numCells;
5622     }
5623   }
5624   /* Build point sf */
5625   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
5626   PetscCall(PetscLayoutSetSize(layout, NVertices));
5627   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
5628   PetscCall(PetscLayoutSetBlockSize(layout, 1));
5629   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
5630   PetscCall(PetscLayoutDestroy(&layout));
5631   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
5632   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
5633   if (dm->sf) {
5634     const char *prefix;
5635 
5636     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
5637     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
5638   }
5639   PetscCall(DMSetPointSF(dm, sfPoint));
5640   PetscCall(PetscSFDestroy(&sfPoint));
5641   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)*vertexSF, "Vertex Ownership SF"));
5642   /* Fill in the rest of the topology structure */
5643   PetscCall(DMPlexSymmetrize(dm));
5644   PetscCall(DMPlexStratify(dm));
5645   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5646   PetscFunctionReturn(PETSC_SUCCESS);
5647 }
5648 
5649 /*@
5650   DMPlexBuildCoordinatesFromCellListParallel - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
5651 
5652   Collective; No Fortran Support
5653 
5654   Input Parameters:
5655 + dm           - The `DM`
5656 . spaceDim     - The spatial dimension used for coordinates
5657 . sfVert       - `PetscSF` describing complete vertex ownership
5658 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
5659 
5660   Level: advanced
5661 
5662 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellListParallel()`
5663 @*/
5664 PetscErrorCode DMPlexBuildCoordinatesFromCellListParallel(DM dm, PetscInt spaceDim, PetscSF sfVert, const PetscReal vertexCoords[])
5665 {
5666   PetscSection coordSection;
5667   Vec          coordinates;
5668   PetscScalar *coords;
5669   PetscInt     numVertices, numVerticesAdj, coordSize, v, vStart, vEnd;
5670   PetscMPIInt  spaceDimi;
5671 
5672   PetscFunctionBegin;
5673   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
5674   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
5675   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
5676   PetscCall(DMSetCoordinateDim(dm, spaceDim));
5677   PetscCall(PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL));
5678   PetscCheck(vEnd - vStart == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Supplied sfVert has wrong number of leaves = %" PetscInt_FMT " != %" PetscInt_FMT " = vEnd - vStart", numVerticesAdj, vEnd - vStart);
5679   PetscCall(DMGetCoordinateSection(dm, &coordSection));
5680   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5681   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
5682   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
5683   for (v = vStart; v < vEnd; ++v) {
5684     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
5685     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
5686   }
5687   PetscCall(PetscSectionSetUp(coordSection));
5688   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
5689   PetscCall(VecCreate(PetscObjectComm((PetscObject)dm), &coordinates));
5690   PetscCall(VecSetBlockSize(coordinates, spaceDim));
5691   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5692   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
5693   PetscCall(VecSetType(coordinates, VECSTANDARD));
5694   PetscCall(VecGetArray(coordinates, &coords));
5695   {
5696     MPI_Datatype coordtype;
5697 
5698     /* Need a temp buffer for coords if we have complex/single */
5699     PetscCall(PetscMPIIntCast(spaceDim, &spaceDimi));
5700     PetscCallMPI(MPI_Type_contiguous(spaceDimi, MPIU_SCALAR, &coordtype));
5701     PetscCallMPI(MPI_Type_commit(&coordtype));
5702 #if defined(PETSC_USE_COMPLEX)
5703     {
5704       PetscScalar *svertexCoords;
5705       PetscInt     i;
5706       PetscCall(PetscMalloc1(numVertices * spaceDim, &svertexCoords));
5707       for (i = 0; i < numVertices * spaceDim; i++) svertexCoords[i] = vertexCoords[i];
5708       PetscCall(PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
5709       PetscCall(PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
5710       PetscCall(PetscFree(svertexCoords));
5711     }
5712 #else
5713     PetscCall(PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
5714     PetscCall(PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
5715 #endif
5716     PetscCallMPI(MPI_Type_free(&coordtype));
5717   }
5718   PetscCall(VecRestoreArray(coordinates, &coords));
5719   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
5720   PetscCall(VecDestroy(&coordinates));
5721   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
5722   PetscFunctionReturn(PETSC_SUCCESS);
5723 }
5724 
5725 /*@
5726   DMPlexCreateFromCellListParallelPetsc - Create distributed `DMPLEX` from a list of vertices for each cell (common mesh generator output) where all cells have the same celltype
5727 
5728   Collective
5729 
5730   Input Parameters:
5731 + comm         - The communicator
5732 . dim          - The topological dimension of the mesh
5733 . numCells     - The number of cells owned by this process
5734 . numVertices  - The number of vertices owned by this process, or `PETSC_DECIDE`
5735 . NVertices    - The global number of vertices, or `PETSC_DECIDE`
5736 . numCorners   - The number of vertices for each cell
5737 . interpolate  - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
5738 . cells        - An array of numCells*numCorners numbers, the global vertex numbers for each cell
5739 . spaceDim     - The spatial dimension used for coordinates
5740 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
5741 
5742   Output Parameters:
5743 + dm          - The `DM`
5744 . vertexSF    - (Optional) `PetscSF` describing complete vertex ownership
5745 - verticesAdj - (Optional) vertex adjacency array
5746 
5747   Level: intermediate
5748 
5749   Notes:
5750   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`,
5751   `DMPlexBuildFromCellListParallel()`, `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellListParallel()`
5752 
5753   See `DMPlexBuildFromCellListParallel()` for an example and details about the topology-related parameters.
5754 
5755   See `DMPlexBuildCoordinatesFromCellListParallel()` for details about the geometry-related parameters.
5756 
5757 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
5758 @*/
5759 PetscErrorCode DMPlexCreateFromCellListParallelPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, PetscInt **verticesAdj, DM *dm)
5760 {
5761   PetscSF sfVert;
5762 
5763   PetscFunctionBegin;
5764   PetscCall(DMCreate(comm, dm));
5765   PetscCall(DMSetType(*dm, DMPLEX));
5766   PetscValidLogicalCollectiveInt(*dm, dim, 2);
5767   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
5768   PetscCall(DMSetDimension(*dm, dim));
5769   PetscCall(DMPlexBuildFromCellListParallel(*dm, numCells, numVertices, NVertices, numCorners, cells, &sfVert, verticesAdj));
5770   if (interpolate) {
5771     DM idm;
5772 
5773     PetscCall(DMPlexInterpolate(*dm, &idm));
5774     PetscCall(DMDestroy(dm));
5775     *dm = idm;
5776   }
5777   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
5778   if (vertexSF) *vertexSF = sfVert;
5779   else PetscCall(PetscSFDestroy(&sfVert));
5780   PetscFunctionReturn(PETSC_SUCCESS);
5781 }
5782 
5783 /*@
5784   DMPlexCreateFromCellSectionParallel - Create distributed `DMPLEX` from a list of vertices for each cell (common mesh generator output) and supports multiple celltypes
5785 
5786   Collective
5787 
5788   Input Parameters:
5789 + comm         - The communicator
5790 . dim          - The topological dimension of the mesh
5791 . numCells     - The number of cells owned by this process
5792 . numVertices  - The number of vertices owned by this process, or `PETSC_DECIDE`
5793 . NVertices    - The global number of vertices, or `PETSC_DECIDE`
5794 . cellSection  - The `PetscSection` giving the number of vertices for each cell (layout of cells)
5795 . interpolate  - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
5796 . cells        - An array of the global vertex numbers for each cell
5797 . spaceDim     - The spatial dimension used for coordinates
5798 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
5799 
5800   Output Parameters:
5801 + dm          - The `DM`
5802 . vertexSF    - (Optional) `PetscSF` describing complete vertex ownership
5803 - verticesAdj - (Optional) vertex adjacency array
5804 
5805   Level: intermediate
5806 
5807   Notes:
5808   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`,
5809   `DMPlexBuildFromCellSectionParallel()`, `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellListParallel()`
5810 
5811   See `DMPlexBuildFromCellSectionParallel()` for an example and details about the topology-related parameters.
5812 
5813   See `DMPlexBuildCoordinatesFromCellListParallel()` for details about the geometry-related parameters.
5814 
5815 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
5816 @*/
5817 PetscErrorCode DMPlexCreateFromCellSectionParallel(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscSection cellSection, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, PetscInt **verticesAdj, DM *dm)
5818 {
5819   PetscSF sfVert;
5820 
5821   PetscFunctionBegin;
5822   PetscCall(DMCreate(comm, dm));
5823   PetscCall(DMSetType(*dm, DMPLEX));
5824   PetscValidLogicalCollectiveInt(*dm, dim, 2);
5825   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
5826   PetscCall(DMSetDimension(*dm, dim));
5827   PetscCall(DMPlexBuildFromCellSectionParallel(*dm, numCells, numVertices, NVertices, cellSection, cells, &sfVert, verticesAdj));
5828   if (interpolate) {
5829     DM idm;
5830 
5831     PetscCall(DMPlexInterpolate(*dm, &idm));
5832     PetscCall(DMDestroy(dm));
5833     *dm = idm;
5834   }
5835   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
5836   if (vertexSF) *vertexSF = sfVert;
5837   else PetscCall(PetscSFDestroy(&sfVert));
5838   PetscFunctionReturn(PETSC_SUCCESS);
5839 }
5840 
5841 /*@
5842   DMPlexBuildFromCellList - Build `DMPLEX` topology from a list of vertices for each cell (common mesh generator output)
5843 
5844   Collective; No Fortran Support
5845 
5846   Input Parameters:
5847 + dm          - The `DM`
5848 . numCells    - The number of cells owned by this process
5849 . numVertices - The number of vertices owned by this process, or `PETSC_DETERMINE`
5850 . numCorners  - The number of vertices for each cell
5851 - cells       - An array of `numCells` x `numCorners` numbers, the global vertex numbers for each cell
5852 
5853   Level: advanced
5854 
5855   Notes:
5856   Two triangles sharing a face
5857 .vb
5858 
5859         2
5860       / | \
5861      /  |  \
5862     /   |   \
5863    0  0 | 1  3
5864     \   |   /
5865      \  |  /
5866       \ | /
5867         1
5868 .ve
5869   would have input
5870 .vb
5871   numCells = 2, numVertices = 4
5872   cells = [0 1 2  1 3 2]
5873 .ve
5874   which would result in the `DMPLEX`
5875 .vb
5876 
5877         4
5878       / | \
5879      /  |  \
5880     /   |   \
5881    2  0 | 1  5
5882     \   |   /
5883      \  |  /
5884       \ | /
5885         3
5886 .ve
5887 
5888   If numVertices is `PETSC_DETERMINE`, it is computed by PETSc as the maximum vertex index in cells + 1.
5889 
5890 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListPetsc()`
5891 @*/
5892 PetscErrorCode DMPlexBuildFromCellList(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const PetscInt cells[])
5893 {
5894   PetscInt *cones, c, p, dim;
5895 
5896   PetscFunctionBegin;
5897   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5898   PetscCall(DMGetDimension(dm, &dim));
5899   /* Get/check global number of vertices */
5900   {
5901     PetscInt       NVerticesInCells, i;
5902     const PetscInt len = numCells * numCorners;
5903 
5904     /* NVerticesInCells = max(cells) + 1 */
5905     NVerticesInCells = PETSC_INT_MIN;
5906     for (i = 0; i < len; i++)
5907       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
5908     ++NVerticesInCells;
5909 
5910     if (numVertices == PETSC_DECIDE) numVertices = NVerticesInCells;
5911     else
5912       PetscCheck(numVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, numVertices, NVerticesInCells);
5913   }
5914   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
5915   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
5916   PetscCall(DMSetUp(dm));
5917   PetscCall(DMPlexGetCones(dm, &cones));
5918   for (c = 0; c < numCells; ++c) {
5919     for (p = 0; p < numCorners; ++p) cones[c * numCorners + p] = cells[c * numCorners + p] + numCells;
5920   }
5921   PetscCall(DMPlexSymmetrize(dm));
5922   PetscCall(DMPlexStratify(dm));
5923   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5924   PetscFunctionReturn(PETSC_SUCCESS);
5925 }
5926 
5927 /*@
5928   DMPlexBuildCoordinatesFromCellList - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
5929 
5930   Collective
5931 
5932   Input Parameters:
5933 + dm           - The `DM`
5934 . spaceDim     - The spatial dimension used for coordinates
5935 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
5936 
5937   Level: advanced
5938 
5939 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellList()`
5940 @*/
5941 PetscErrorCode DMPlexBuildCoordinatesFromCellList(DM dm, PetscInt spaceDim, const PetscReal vertexCoords[])
5942 {
5943   PetscSection coordSection;
5944   Vec          coordinates;
5945   DM           cdm;
5946   PetscScalar *coords;
5947   PetscInt     v, vStart, vEnd, d;
5948 
5949   PetscFunctionBegin;
5950   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
5951   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
5952   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
5953   PetscCall(DMSetCoordinateDim(dm, spaceDim));
5954   PetscCall(DMGetCoordinateSection(dm, &coordSection));
5955   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5956   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
5957   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
5958   for (v = vStart; v < vEnd; ++v) {
5959     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
5960     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
5961   }
5962   PetscCall(PetscSectionSetUp(coordSection));
5963 
5964   PetscCall(DMGetCoordinateDM(dm, &cdm));
5965   PetscCall(DMCreateLocalVector(cdm, &coordinates));
5966   PetscCall(VecSetBlockSize(coordinates, spaceDim));
5967   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5968   PetscCall(VecGetArrayWrite(coordinates, &coords));
5969   for (v = 0; v < vEnd - vStart; ++v) {
5970     for (d = 0; d < spaceDim; ++d) coords[v * spaceDim + d] = vertexCoords[v * spaceDim + d];
5971   }
5972   PetscCall(VecRestoreArrayWrite(coordinates, &coords));
5973   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
5974   PetscCall(VecDestroy(&coordinates));
5975   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
5976   PetscFunctionReturn(PETSC_SUCCESS);
5977 }
5978 
5979 /*@
5980   DMPlexCreateFromCellListPetsc - Create `DMPLEX` from a list of vertices for each cell (common mesh generator output), but only process 0 takes in the input
5981 
5982   Collective
5983 
5984   Input Parameters:
5985 + comm         - The communicator
5986 . dim          - The topological dimension of the mesh
5987 . numCells     - The number of cells, only on process 0
5988 . numVertices  - The number of vertices owned by this process, or `PETSC_DECIDE`, only on process 0
5989 . numCorners   - The number of vertices for each cell, only on process 0
5990 . interpolate  - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
5991 . cells        - An array of numCells*numCorners numbers, the vertices for each cell, only on process 0
5992 . spaceDim     - The spatial dimension used for coordinates
5993 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex, only on process 0
5994 
5995   Output Parameter:
5996 . dm - The `DM`, which only has points on process 0
5997 
5998   Level: intermediate
5999 
6000   Notes:
6001   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`, `DMPlexBuildFromCellList()`,
6002   `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellList()`
6003 
6004   See `DMPlexBuildFromCellList()` for an example and details about the topology-related parameters.
6005   See `DMPlexBuildCoordinatesFromCellList()` for details about the geometry-related parameters.
6006   See `DMPlexCreateFromCellListParallelPetsc()` for parallel input
6007 
6008 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellList()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
6009 @*/
6010 PetscErrorCode DMPlexCreateFromCellListPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], DM *dm)
6011 {
6012   PetscMPIInt rank;
6013 
6014   PetscFunctionBegin;
6015   PetscCheck(dim, comm, PETSC_ERR_ARG_OUTOFRANGE, "This is not appropriate for 0-dimensional meshes. Consider either creating the DM using DMPlexCreateFromDAG(), by hand, or using DMSwarm.");
6016   PetscCallMPI(MPI_Comm_rank(comm, &rank));
6017   PetscCall(DMCreate(comm, dm));
6018   PetscCall(DMSetType(*dm, DMPLEX));
6019   PetscCall(DMSetDimension(*dm, dim));
6020   if (rank == 0) PetscCall(DMPlexBuildFromCellList(*dm, numCells, numVertices, numCorners, cells));
6021   else PetscCall(DMPlexBuildFromCellList(*dm, 0, 0, 0, NULL));
6022   if (interpolate) {
6023     DM idm;
6024 
6025     PetscCall(DMPlexInterpolate(*dm, &idm));
6026     PetscCall(DMDestroy(dm));
6027     *dm = idm;
6028   }
6029   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, vertexCoords));
6030   else PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, NULL));
6031   PetscFunctionReturn(PETSC_SUCCESS);
6032 }
6033 
6034 /*@
6035   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a `DM`
6036 
6037   Input Parameters:
6038 + dm               - The empty `DM` object, usually from `DMCreate()` and `DMSetDimension()`
6039 . depth            - The depth of the DAG
6040 . numPoints        - Array of size depth + 1 containing the number of points at each `depth`
6041 . coneSize         - The cone size of each point
6042 . cones            - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
6043 . coneOrientations - The orientation of each cone point
6044 - vertexCoords     - An array of `numPoints`[0]*spacedim numbers representing the coordinates of each vertex, with spacedim the value set via `DMSetCoordinateDim()`
6045 
6046   Output Parameter:
6047 . dm - The `DM`
6048 
6049   Level: advanced
6050 
6051   Note:
6052   Two triangles sharing a face would have input
6053 .vb
6054   depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
6055   cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
6056  vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
6057 .ve
6058   which would result in the DMPlex
6059 .vb
6060         4
6061       / | \
6062      /  |  \
6063     /   |   \
6064    2  0 | 1  5
6065     \   |   /
6066      \  |  /
6067       \ | /
6068         3
6069 .ve
6070   Notice that all points are numbered consecutively, unlike `DMPlexCreateFromCellListPetsc()`
6071 
6072 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
6073 @*/
6074 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
6075 {
6076   Vec          coordinates;
6077   PetscSection coordSection;
6078   PetscScalar *coords;
6079   PetscInt     coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
6080 
6081   PetscFunctionBegin;
6082   PetscCall(DMGetDimension(dm, &dim));
6083   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
6084   PetscCheck(dimEmbed >= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Embedding dimension %" PetscInt_FMT " cannot be less than intrinsic dimension %" PetscInt_FMT, dimEmbed, dim);
6085   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
6086   PetscCall(DMPlexSetChart(dm, pStart, pEnd));
6087   for (p = pStart; p < pEnd; ++p) {
6088     PetscCall(DMPlexSetConeSize(dm, p, coneSize[p - pStart]));
6089     if (firstVertex < 0 && !coneSize[p - pStart]) firstVertex = p - pStart;
6090   }
6091   PetscCheck(firstVertex >= 0 || !numPoints[0], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Expected %" PetscInt_FMT " vertices but could not find any", numPoints[0]);
6092   PetscCall(DMSetUp(dm)); /* Allocate space for cones */
6093   for (p = pStart, off = 0; p < pEnd; off += coneSize[p - pStart], ++p) {
6094     PetscCall(DMPlexSetCone(dm, p, &cones[off]));
6095     PetscCall(DMPlexSetConeOrientation(dm, p, &coneOrientations[off]));
6096   }
6097   PetscCall(DMPlexSymmetrize(dm));
6098   PetscCall(DMPlexStratify(dm));
6099   /* Build coordinates */
6100   PetscCall(DMGetCoordinateSection(dm, &coordSection));
6101   PetscCall(PetscSectionSetNumFields(coordSection, 1));
6102   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dimEmbed));
6103   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numPoints[0]));
6104   for (v = firstVertex; v < firstVertex + numPoints[0]; ++v) {
6105     PetscCall(PetscSectionSetDof(coordSection, v, dimEmbed));
6106     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed));
6107   }
6108   PetscCall(PetscSectionSetUp(coordSection));
6109   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
6110   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
6111   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
6112   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
6113   PetscCall(VecSetBlockSize(coordinates, dimEmbed));
6114   PetscCall(VecSetType(coordinates, VECSTANDARD));
6115   if (vertexCoords) {
6116     PetscCall(VecGetArray(coordinates, &coords));
6117     for (v = 0; v < numPoints[0]; ++v) {
6118       PetscInt off;
6119 
6120       PetscCall(PetscSectionGetOffset(coordSection, v + firstVertex, &off));
6121       for (d = 0; d < dimEmbed; ++d) coords[off + d] = vertexCoords[v * dimEmbed + d];
6122     }
6123   }
6124   PetscCall(VecRestoreArray(coordinates, &coords));
6125   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
6126   PetscCall(VecDestroy(&coordinates));
6127   PetscFunctionReturn(PETSC_SUCCESS);
6128 }
6129 
6130 /*
6131   DMPlexCreateCellVertexFromFile - Create a `DMPLEX` mesh from a simple cell-vertex file.
6132 
6133   Collective
6134 
6135 + comm        - The MPI communicator
6136 . filename    - Name of the .dat file
6137 - interpolate - Create faces and edges in the mesh
6138 
6139   Output Parameter:
6140 . dm  - The `DM` object representing the mesh
6141 
6142   Level: beginner
6143 
6144   Note:
6145   The format is the simplest possible:
6146 .vb
6147   dim Ne Nv Nc Nl
6148   v_1 v_2 ... v_Nc
6149   ...
6150   x y z marker_1 ... marker_Nl
6151 .ve
6152 
6153   Developer Note:
6154   Should use a `PetscViewer` not a filename
6155 
6156 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromFile()`, `DMPlexCreateGmsh()`, `DMPlexCreate()`
6157 */
6158 static PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
6159 {
6160   DMLabel      marker;
6161   PetscViewer  viewer;
6162   Vec          coordinates;
6163   PetscSection coordSection;
6164   PetscScalar *coords;
6165   char         line[PETSC_MAX_PATH_LEN];
6166   PetscInt     cdim, coordSize, v, c, d;
6167   PetscMPIInt  rank;
6168   int          snum, dim, Nv, Nc, Ncn, Nl;
6169 
6170   PetscFunctionBegin;
6171   PetscCallMPI(MPI_Comm_rank(comm, &rank));
6172   PetscCall(PetscViewerCreate(comm, &viewer));
6173   PetscCall(PetscViewerSetType(viewer, PETSCVIEWERASCII));
6174   PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
6175   PetscCall(PetscViewerFileSetName(viewer, filename));
6176   if (rank == 0) {
6177     PetscCall(PetscViewerRead(viewer, line, 5, NULL, PETSC_STRING));
6178     snum = sscanf(line, "%d %d %d %d %d", &dim, &Nc, &Nv, &Ncn, &Nl);
6179     PetscCheck(snum == 5, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
6180   } else {
6181     Nc = Nv = Ncn = Nl = 0;
6182   }
6183   PetscCallMPI(MPI_Bcast(&dim, 1, MPI_INT, 0, comm));
6184   cdim = dim;
6185   PetscCall(DMCreate(comm, dm));
6186   PetscCall(DMSetType(*dm, DMPLEX));
6187   PetscCall(DMPlexSetChart(*dm, 0, Nc + Nv));
6188   PetscCall(DMSetDimension(*dm, dim));
6189   PetscCall(DMSetCoordinateDim(*dm, cdim));
6190   /* Read topology */
6191   if (rank == 0) {
6192     char     format[PETSC_MAX_PATH_LEN];
6193     PetscInt cone[8];
6194     int      vbuf[8], v;
6195 
6196     for (c = 0; c < Ncn; ++c) {
6197       format[c * 3 + 0] = '%';
6198       format[c * 3 + 1] = 'd';
6199       format[c * 3 + 2] = ' ';
6200     }
6201     format[Ncn * 3 - 1] = '\0';
6202     for (c = 0; c < Nc; ++c) PetscCall(DMPlexSetConeSize(*dm, c, Ncn));
6203     PetscCall(DMSetUp(*dm));
6204     for (c = 0; c < Nc; ++c) {
6205       PetscCall(PetscViewerRead(viewer, line, Ncn, NULL, PETSC_STRING));
6206       switch (Ncn) {
6207       case 2:
6208         snum = sscanf(line, format, &vbuf[0], &vbuf[1]);
6209         break;
6210       case 3:
6211         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2]);
6212         break;
6213       case 4:
6214         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3]);
6215         break;
6216       case 6:
6217         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5]);
6218         break;
6219       case 8:
6220         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);
6221         break;
6222       default:
6223         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No cell shape with %d vertices", Ncn);
6224       }
6225       PetscCheck(snum == Ncn, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
6226       for (v = 0; v < Ncn; ++v) cone[v] = vbuf[v] + Nc;
6227       /* Hexahedra are inverted */
6228       if (Ncn == 8) {
6229         PetscInt tmp = cone[1];
6230         cone[1]      = cone[3];
6231         cone[3]      = tmp;
6232       }
6233       PetscCall(DMPlexSetCone(*dm, c, cone));
6234     }
6235   }
6236   PetscCall(DMPlexSymmetrize(*dm));
6237   PetscCall(DMPlexStratify(*dm));
6238   /* Read coordinates */
6239   PetscCall(DMGetCoordinateSection(*dm, &coordSection));
6240   PetscCall(PetscSectionSetNumFields(coordSection, 1));
6241   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
6242   PetscCall(PetscSectionSetChart(coordSection, Nc, Nc + Nv));
6243   for (v = Nc; v < Nc + Nv; ++v) {
6244     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
6245     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
6246   }
6247   PetscCall(PetscSectionSetUp(coordSection));
6248   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
6249   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
6250   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
6251   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
6252   PetscCall(VecSetBlockSize(coordinates, cdim));
6253   PetscCall(VecSetType(coordinates, VECSTANDARD));
6254   PetscCall(VecGetArray(coordinates, &coords));
6255   if (rank == 0) {
6256     char   format[PETSC_MAX_PATH_LEN];
6257     double x[3];
6258     int    l, val[3];
6259 
6260     if (Nl) {
6261       for (l = 0; l < Nl; ++l) {
6262         format[l * 3 + 0] = '%';
6263         format[l * 3 + 1] = 'd';
6264         format[l * 3 + 2] = ' ';
6265       }
6266       format[Nl * 3 - 1] = '\0';
6267       PetscCall(DMCreateLabel(*dm, "marker"));
6268       PetscCall(DMGetLabel(*dm, "marker", &marker));
6269     }
6270     for (v = 0; v < Nv; ++v) {
6271       PetscCall(PetscViewerRead(viewer, line, 3 + Nl, NULL, PETSC_STRING));
6272       snum = sscanf(line, "%lg %lg %lg", &x[0], &x[1], &x[2]);
6273       PetscCheck(snum == 3, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
6274       switch (Nl) {
6275       case 0:
6276         snum = 0;
6277         break;
6278       case 1:
6279         snum = sscanf(line, format, &val[0]);
6280         break;
6281       case 2:
6282         snum = sscanf(line, format, &val[0], &val[1]);
6283         break;
6284       case 3:
6285         snum = sscanf(line, format, &val[0], &val[1], &val[2]);
6286         break;
6287       default:
6288         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Request support for %d labels", Nl);
6289       }
6290       PetscCheck(snum == Nl, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
6291       for (d = 0; d < cdim; ++d) coords[v * cdim + d] = x[d];
6292       for (l = 0; l < Nl; ++l) PetscCall(DMLabelSetValue(marker, v + Nc, val[l]));
6293     }
6294   }
6295   PetscCall(VecRestoreArray(coordinates, &coords));
6296   PetscCall(DMSetCoordinatesLocal(*dm, coordinates));
6297   PetscCall(VecDestroy(&coordinates));
6298   PetscCall(PetscViewerDestroy(&viewer));
6299   if (interpolate) {
6300     DM      idm;
6301     DMLabel bdlabel;
6302 
6303     PetscCall(DMPlexInterpolate(*dm, &idm));
6304     PetscCall(DMDestroy(dm));
6305     *dm = idm;
6306 
6307     if (!Nl) {
6308       PetscCall(DMCreateLabel(*dm, "marker"));
6309       PetscCall(DMGetLabel(*dm, "marker", &bdlabel));
6310       PetscCall(DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel));
6311       PetscCall(DMPlexLabelComplete(*dm, bdlabel));
6312     }
6313   }
6314   PetscFunctionReturn(PETSC_SUCCESS);
6315 }
6316 
6317 /*@
6318   DMPlexCreateFromFile - This takes a filename and produces a `DM`
6319 
6320   Collective
6321 
6322   Input Parameters:
6323 + comm        - The communicator
6324 . filename    - A file name
6325 . plexname    - The object name of the resulting `DM`, also used for intra-datafile lookup by some formats
6326 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
6327 
6328   Output Parameter:
6329 . dm - The `DM`
6330 
6331   Options Database Key:
6332 . -dm_plex_create_from_hdf5_xdmf - use the `PETSC_VIEWER_HDF5_XDMF` format for reading HDF5
6333 
6334   Use `-dm_plex_create_ prefix` to pass options to the internal `PetscViewer`, e.g.
6335 $ -dm_plex_create_viewer_hdf5_collective
6336 
6337   Level: beginner
6338 
6339   Notes:
6340   Using `PETSCVIEWERHDF5` type with `PETSC_VIEWER_HDF5_PETSC` format, one can save multiple `DMPLEX`
6341   meshes in a single HDF5 file. This in turn requires one to name the `DMPLEX` object with `PetscObjectSetName()`
6342   before saving it with `DMView()` and before loading it with `DMLoad()` for identification of the mesh object.
6343   The input parameter name is thus used to name the `DMPLEX` object when `DMPlexCreateFromFile()` internally
6344   calls `DMLoad()`. Currently, name is ignored for other viewer types and/or formats.
6345 
6346 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`, `PetscObjectSetName()`, `DMView()`, `DMLoad()`
6347 @*/
6348 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], const char plexname[], PetscBool interpolate, DM *dm)
6349 {
6350   const char  extGmsh[]      = ".msh";
6351   const char  extGmsh2[]     = ".msh2";
6352   const char  extGmsh4[]     = ".msh4";
6353   const char  extCGNS[]      = ".cgns";
6354   const char  extExodus[]    = ".exo";
6355   const char  extExodus_e[]  = ".e";
6356   const char  extGenesis[]   = ".gen";
6357   const char  extFluent[]    = ".cas";
6358   const char  extHDF5[]      = ".h5";
6359   const char  extXDMFHDF5[]  = ".xdmf.h5";
6360   const char  extPLY[]       = ".ply";
6361   const char  extEGADSLite[] = ".egadslite";
6362   const char  extEGADS[]     = ".egads";
6363   const char  extIGES[]      = ".igs";
6364   const char  extSTEP[]      = ".stp";
6365   const char  extCV[]        = ".dat";
6366   size_t      len;
6367   PetscBool   isGmsh, isGmsh2, isGmsh4, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isPLY, isEGADSLite, isEGADS, isIGES, isSTEP, isCV, isXDMFHDF5;
6368   PetscMPIInt rank;
6369 
6370   PetscFunctionBegin;
6371   PetscAssertPointer(filename, 2);
6372   if (plexname) PetscAssertPointer(plexname, 3);
6373   PetscAssertPointer(dm, 5);
6374   PetscCall(DMInitializePackage());
6375   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromFile, 0, 0, 0, 0));
6376   PetscCallMPI(MPI_Comm_rank(comm, &rank));
6377   PetscCall(PetscStrlen(filename, &len));
6378   PetscCheck(len, comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
6379 
6380 #define CheckExtension(extension__, is_extension__) \
6381   do { \
6382     PetscAssert(sizeof(extension__), comm, PETSC_ERR_PLIB, "Zero-size extension: %s", extension__); \
6383     /* don't count the null-terminator at the end */ \
6384     const size_t ext_len = sizeof(extension__) - 1; \
6385     if (len < ext_len) { \
6386       is_extension__ = PETSC_FALSE; \
6387     } else { \
6388       PetscCall(PetscStrncmp(filename + len - ext_len, extension__, ext_len, &is_extension__)); \
6389     } \
6390   } while (0)
6391 
6392   CheckExtension(extGmsh, isGmsh);
6393   CheckExtension(extGmsh2, isGmsh2);
6394   CheckExtension(extGmsh4, isGmsh4);
6395   CheckExtension(extCGNS, isCGNS);
6396   CheckExtension(extExodus, isExodus);
6397   if (!isExodus) CheckExtension(extExodus_e, isExodus);
6398   CheckExtension(extGenesis, isGenesis);
6399   CheckExtension(extFluent, isFluent);
6400   CheckExtension(extHDF5, isHDF5);
6401   CheckExtension(extPLY, isPLY);
6402   CheckExtension(extEGADSLite, isEGADSLite);
6403   CheckExtension(extEGADS, isEGADS);
6404   CheckExtension(extIGES, isIGES);
6405   CheckExtension(extSTEP, isSTEP);
6406   CheckExtension(extCV, isCV);
6407   CheckExtension(extXDMFHDF5, isXDMFHDF5);
6408 
6409 #undef CheckExtension
6410 
6411   if (isGmsh || isGmsh2 || isGmsh4) {
6412     PetscCall(DMPlexCreateGmshFromFile(comm, filename, interpolate, dm));
6413   } else if (isCGNS) {
6414     PetscCall(DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm));
6415   } else if (isExodus || isGenesis) {
6416     PetscCall(DMPlexCreateExodusFromFile(comm, filename, interpolate, dm));
6417   } else if (isFluent) {
6418     PetscCall(DMPlexCreateFluentFromFile(comm, filename, interpolate, dm));
6419   } else if (isHDF5) {
6420     PetscViewer viewer;
6421 
6422     /* PETSC_VIEWER_HDF5_XDMF is used if the filename ends with .xdmf.h5, or if -dm_plex_create_from_hdf5_xdmf option is present */
6423     PetscCall(PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &isXDMFHDF5, NULL));
6424     PetscCall(PetscViewerCreate(comm, &viewer));
6425     PetscCall(PetscViewerSetType(viewer, PETSCVIEWERHDF5));
6426     PetscCall(PetscViewerSetOptionsPrefix(viewer, "dm_plex_create_"));
6427     PetscCall(PetscViewerSetFromOptions(viewer));
6428     PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
6429     PetscCall(PetscViewerFileSetName(viewer, filename));
6430 
6431     PetscCall(DMCreate(comm, dm));
6432     PetscCall(PetscObjectSetName((PetscObject)*dm, plexname));
6433     PetscCall(DMSetType(*dm, DMPLEX));
6434     if (isXDMFHDF5) PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF));
6435     PetscCall(DMLoad(*dm, viewer));
6436     if (isXDMFHDF5) PetscCall(PetscViewerPopFormat(viewer));
6437     PetscCall(PetscViewerDestroy(&viewer));
6438 
6439     if (interpolate) {
6440       DM idm;
6441 
6442       PetscCall(DMPlexInterpolate(*dm, &idm));
6443       PetscCall(DMDestroy(dm));
6444       *dm = idm;
6445     }
6446   } else if (isPLY) {
6447     PetscCall(DMPlexCreatePLYFromFile(comm, filename, interpolate, dm));
6448   } else if (isEGADSLite || isEGADS || isIGES || isSTEP) {
6449     if (isEGADSLite) PetscCall(DMPlexCreateEGADSLiteFromFile(comm, filename, dm));
6450     else PetscCall(DMPlexCreateEGADSFromFile(comm, filename, dm));
6451     if (!interpolate) {
6452       DM udm;
6453 
6454       PetscCall(DMPlexUninterpolate(*dm, &udm));
6455       PetscCall(DMDestroy(dm));
6456       *dm = udm;
6457     }
6458   } else if (isCV) {
6459     PetscCall(DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm));
6460   } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
6461   PetscCall(PetscStrlen(plexname, &len));
6462   if (len) PetscCall(PetscObjectSetName((PetscObject)*dm, plexname));
6463   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromFile, 0, 0, 0, 0));
6464   PetscFunctionReturn(PETSC_SUCCESS);
6465 }
6466 
6467 /*@
6468   DMPlexCreateEphemeral - This takes a `DMPlexTransform` and a base `DMPlex` and produces an ephemeral `DM`, meaning one that is created on the fly in response to queries.
6469 
6470   Input Parameters:
6471 + tr     - The `DMPlexTransform`
6472 - prefix - An options prefix, or NULL
6473 
6474   Output Parameter:
6475 . dm - The `DM`
6476 
6477   Level: beginner
6478 
6479   Notes:
6480   An emphemeral mesh is one that is not stored concretely, as in the default `DMPLEX` implementation, but rather is produced on the fly in response to queries, using information from the transform and the base mesh.
6481 
6482 .seealso: `DMPlexCreateFromFile`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
6483 @*/
6484 PetscErrorCode DMPlexCreateEphemeral(DMPlexTransform tr, const char prefix[], DM *dm)
6485 {
6486   DM           bdm, bcdm, cdm;
6487   Vec          coordinates, coordinatesNew;
6488   PetscSection cs;
6489   PetscInt     cdim, Nl;
6490 
6491   PetscFunctionBegin;
6492   PetscCall(DMCreate(PetscObjectComm((PetscObject)tr), dm));
6493   PetscCall(DMSetType(*dm, DMPLEX));
6494   ((DM_Plex *)(*dm)->data)->interpolated = DMPLEX_INTERPOLATED_FULL;
6495   // Handle coordinates
6496   PetscCall(DMPlexTransformGetDM(tr, &bdm));
6497   PetscCall(DMPlexTransformSetDimensions(tr, bdm, *dm));
6498   PetscCall(DMGetCoordinateDim(*dm, &cdim));
6499   PetscCall(DMGetCoordinateDM(bdm, &bcdm));
6500   PetscCall(DMGetCoordinateDM(*dm, &cdm));
6501   PetscCall(DMCopyDisc(bcdm, cdm));
6502   PetscCall(DMGetLocalSection(cdm, &cs));
6503   PetscCall(PetscSectionSetNumFields(cs, 1));
6504   PetscCall(PetscSectionSetFieldComponents(cs, 0, cdim));
6505   PetscCall(DMGetCoordinatesLocal(bdm, &coordinates));
6506   PetscCall(VecDuplicate(coordinates, &coordinatesNew));
6507   PetscCall(VecCopy(coordinates, coordinatesNew));
6508   PetscCall(DMSetCoordinatesLocal(*dm, coordinatesNew));
6509   PetscCall(VecDestroy(&coordinatesNew));
6510 
6511   PetscCall(PetscObjectReference((PetscObject)tr));
6512   PetscCall(DMPlexTransformDestroy(&((DM_Plex *)(*dm)->data)->tr));
6513   ((DM_Plex *)(*dm)->data)->tr = tr;
6514   PetscCall(DMPlexDistributeSetDefault(*dm, PETSC_FALSE));
6515   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)*dm, prefix));
6516   PetscCall(DMSetFromOptions(*dm));
6517 
6518   PetscCall(DMGetNumLabels(bdm, &Nl));
6519   for (PetscInt l = 0; l < Nl; ++l) {
6520     DMLabel     label, labelNew;
6521     const char *lname;
6522     PetscBool   isDepth, isCellType;
6523 
6524     PetscCall(DMGetLabelName(bdm, l, &lname));
6525     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
6526     if (isDepth) continue;
6527     PetscCall(PetscStrcmp(lname, "celltype", &isCellType));
6528     if (isCellType) continue;
6529     PetscCall(DMCreateLabel(*dm, lname));
6530     PetscCall(DMGetLabel(bdm, lname, &label));
6531     PetscCall(DMGetLabel(*dm, lname, &labelNew));
6532     PetscCall(DMLabelSetType(labelNew, DMLABELEPHEMERAL));
6533     PetscCall(DMLabelEphemeralSetLabel(labelNew, label));
6534     PetscCall(DMLabelEphemeralSetTransform(labelNew, tr));
6535     PetscCall(DMLabelSetUp(labelNew));
6536   }
6537   PetscFunctionReturn(PETSC_SUCCESS);
6538 }
6539