xref: /petsc/src/dm/impls/plex/plexcreate.c (revision 9e48eb9f905fbc01704defd592b7e78e540d2a26)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h> /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>
4 #include <petscsf.h>
5 #include <petscdmplextransform.h> /*I   "petscdmplextransform.h"   I*/
6 #include <petscdmlabelephemeral.h>
7 #include <petsc/private/kernels/blockmatmult.h>
8 #include <petsc/private/kernels/blockinvert.h>
9 
10 #ifdef PETSC_HAVE_UNISTD_H
11   #include <unistd.h>
12 #endif
13 #include <errno.h>
14 
15 PetscLogEvent DMPLEX_CreateFromFile, DMPLEX_CreateFromOptions, DMPLEX_BuildFromCellList, DMPLEX_BuildCoordinatesFromCellList;
16 
17 /* External function declarations here */
18 static PetscErrorCode DMInitialize_Plex(DM dm);
19 
20 /* This copies internal things in the Plex structure that we generally want when making a new, related Plex */
21 PetscErrorCode DMPlexCopy_Internal(DM dmin, PetscBool copyPeriodicity, PetscBool copyOverlap, DM dmout)
22 {
23   const PetscReal     *maxCell, *Lstart, *L;
24   VecType              vecType;
25   MatType              matType;
26   PetscBool            dist, useCeed;
27   DMReorderDefaultFlag reorder;
28 
29   PetscFunctionBegin;
30   PetscCall(DMGetVecType(dmin, &vecType));
31   PetscCall(DMSetVecType(dmout, vecType));
32   PetscCall(DMGetMatType(dmin, &matType));
33   PetscCall(DMSetMatType(dmout, matType));
34   if (copyPeriodicity) {
35     PetscCall(DMGetPeriodicity(dmin, &maxCell, &Lstart, &L));
36     PetscCall(DMSetPeriodicity(dmout, maxCell, Lstart, L));
37     PetscCall(DMLocalizeCoordinates(dmout));
38   }
39   PetscCall(DMPlexDistributeGetDefault(dmin, &dist));
40   PetscCall(DMPlexDistributeSetDefault(dmout, dist));
41   PetscCall(DMPlexReorderGetDefault(dmin, &reorder));
42   PetscCall(DMPlexReorderSetDefault(dmout, reorder));
43   PetscCall(DMPlexGetUseCeed(dmin, &useCeed));
44   PetscCall(DMPlexSetUseCeed(dmout, useCeed));
45   ((DM_Plex *)dmout->data)->useHashLocation = ((DM_Plex *)dmin->data)->useHashLocation;
46   ((DM_Plex *)dmout->data)->printSetValues  = ((DM_Plex *)dmin->data)->printSetValues;
47   ((DM_Plex *)dmout->data)->printFEM        = ((DM_Plex *)dmin->data)->printFEM;
48   ((DM_Plex *)dmout->data)->printFVM        = ((DM_Plex *)dmin->data)->printFVM;
49   ((DM_Plex *)dmout->data)->printL2         = ((DM_Plex *)dmin->data)->printL2;
50   ((DM_Plex *)dmout->data)->printLocate     = ((DM_Plex *)dmin->data)->printLocate;
51   ((DM_Plex *)dmout->data)->printTol        = ((DM_Plex *)dmin->data)->printTol;
52   if (copyOverlap) PetscCall(DMPlexSetOverlap_Plex(dmout, dmin, 0));
53   PetscFunctionReturn(PETSC_SUCCESS);
54 }
55 
56 /* Replace dm with the contents of ndm, and then destroy ndm
57    - Share the DM_Plex structure
58    - Share the coordinates
59    - Share the SF
60 */
61 PetscErrorCode DMPlexReplace_Internal(DM dm, DM *ndm)
62 {
63   PetscSF          sf;
64   DM               dmNew = *ndm, coordDM, coarseDM;
65   Vec              coords;
66   const PetscReal *maxCell, *Lstart, *L;
67   PetscInt         dim, cdim;
68 
69   PetscFunctionBegin;
70   if (dm == dmNew) {
71     PetscCall(DMDestroy(ndm));
72     PetscFunctionReturn(PETSC_SUCCESS);
73   }
74   dm->setupcalled = dmNew->setupcalled;
75   if (!dm->hdr.name) {
76     const char *name;
77 
78     PetscCall(PetscObjectGetName((PetscObject)*ndm, &name));
79     PetscCall(PetscObjectSetName((PetscObject)dm, name));
80   }
81   PetscCall(DMGetDimension(dmNew, &dim));
82   PetscCall(DMSetDimension(dm, dim));
83   PetscCall(DMGetCoordinateDim(dmNew, &cdim));
84   PetscCall(DMSetCoordinateDim(dm, cdim));
85   PetscCall(DMGetPointSF(dmNew, &sf));
86   PetscCall(DMSetPointSF(dm, sf));
87   PetscCall(DMGetCoordinateDM(dmNew, &coordDM));
88   PetscCall(DMGetCoordinatesLocal(dmNew, &coords));
89   PetscCall(DMSetCoordinateDM(dm, coordDM));
90   PetscCall(DMSetCoordinatesLocal(dm, coords));
91   PetscCall(DMGetCellCoordinateDM(dmNew, &coordDM));
92   PetscCall(DMGetCellCoordinatesLocal(dmNew, &coords));
93   PetscCall(DMSetCellCoordinateDM(dm, coordDM));
94   PetscCall(DMSetCellCoordinatesLocal(dm, coords));
95   /* Do not want to create the coordinate field if it does not already exist, so do not call DMGetCoordinateField() */
96   PetscCall(DMFieldDestroy(&dm->coordinates[0].field));
97   dm->coordinates[0].field            = dmNew->coordinates[0].field;
98   ((DM_Plex *)dmNew->data)->coordFunc = ((DM_Plex *)dm->data)->coordFunc;
99   PetscCall(DMGetPeriodicity(dmNew, &maxCell, &Lstart, &L));
100   PetscCall(DMSetPeriodicity(dm, maxCell, Lstart, L));
101   PetscCall(DMPlexGetGlobalToNaturalSF(dmNew, &sf));
102   PetscCall(DMPlexSetGlobalToNaturalSF(dm, sf));
103   PetscCall(DMDestroy_Plex(dm));
104   PetscCall(DMInitialize_Plex(dm));
105   dm->data = dmNew->data;
106   ((DM_Plex *)dmNew->data)->refct++;
107   {
108     PetscInt       num_face_sfs;
109     const PetscSF *sfs;
110     PetscCall(DMPlexGetIsoperiodicFaceSF(dm, &num_face_sfs, &sfs));
111     PetscCall(DMPlexSetIsoperiodicFaceSF(dm, num_face_sfs, (PetscSF *)sfs)); // for the compose function effect on dm
112   }
113   PetscCall(DMDestroyLabelLinkList_Internal(dm));
114   PetscCall(DMCopyLabels(dmNew, dm, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL));
115   PetscCall(DMGetCoarseDM(dmNew, &coarseDM));
116   PetscCall(DMSetCoarseDM(dm, coarseDM));
117   PetscCall(DMDestroy(ndm));
118   PetscFunctionReturn(PETSC_SUCCESS);
119 }
120 
121 /* Swap dm with the contents of dmNew
122    - Swap the DM_Plex structure
123    - Swap the coordinates
124    - Swap the point PetscSF
125 */
126 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
127 {
128   DM          coordDMA, coordDMB;
129   Vec         coordsA, coordsB;
130   PetscSF     sfA, sfB;
131   DMField     fieldTmp;
132   void       *tmp;
133   DMLabelLink listTmp;
134   DMLabel     depthTmp;
135   PetscInt    tmpI;
136 
137   PetscFunctionBegin;
138   if (dmA == dmB) PetscFunctionReturn(PETSC_SUCCESS);
139   PetscCall(DMGetPointSF(dmA, &sfA));
140   PetscCall(DMGetPointSF(dmB, &sfB));
141   PetscCall(PetscObjectReference((PetscObject)sfA));
142   PetscCall(DMSetPointSF(dmA, sfB));
143   PetscCall(DMSetPointSF(dmB, sfA));
144   PetscCall(PetscObjectDereference((PetscObject)sfA));
145 
146   PetscCall(DMGetCoordinateDM(dmA, &coordDMA));
147   PetscCall(DMGetCoordinateDM(dmB, &coordDMB));
148   PetscCall(PetscObjectReference((PetscObject)coordDMA));
149   PetscCall(DMSetCoordinateDM(dmA, coordDMB));
150   PetscCall(DMSetCoordinateDM(dmB, coordDMA));
151   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
152 
153   PetscCall(DMGetCoordinatesLocal(dmA, &coordsA));
154   PetscCall(DMGetCoordinatesLocal(dmB, &coordsB));
155   PetscCall(PetscObjectReference((PetscObject)coordsA));
156   PetscCall(DMSetCoordinatesLocal(dmA, coordsB));
157   PetscCall(DMSetCoordinatesLocal(dmB, coordsA));
158   PetscCall(PetscObjectDereference((PetscObject)coordsA));
159 
160   PetscCall(DMGetCellCoordinateDM(dmA, &coordDMA));
161   PetscCall(DMGetCellCoordinateDM(dmB, &coordDMB));
162   PetscCall(PetscObjectReference((PetscObject)coordDMA));
163   PetscCall(DMSetCellCoordinateDM(dmA, coordDMB));
164   PetscCall(DMSetCellCoordinateDM(dmB, coordDMA));
165   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
166 
167   PetscCall(DMGetCellCoordinatesLocal(dmA, &coordsA));
168   PetscCall(DMGetCellCoordinatesLocal(dmB, &coordsB));
169   PetscCall(PetscObjectReference((PetscObject)coordsA));
170   PetscCall(DMSetCellCoordinatesLocal(dmA, coordsB));
171   PetscCall(DMSetCellCoordinatesLocal(dmB, coordsA));
172   PetscCall(PetscObjectDereference((PetscObject)coordsA));
173 
174   fieldTmp                  = dmA->coordinates[0].field;
175   dmA->coordinates[0].field = dmB->coordinates[0].field;
176   dmB->coordinates[0].field = fieldTmp;
177   fieldTmp                  = dmA->coordinates[1].field;
178   dmA->coordinates[1].field = dmB->coordinates[1].field;
179   dmB->coordinates[1].field = fieldTmp;
180   tmp                       = dmA->data;
181   dmA->data                 = dmB->data;
182   dmB->data                 = tmp;
183   listTmp                   = dmA->labels;
184   dmA->labels               = dmB->labels;
185   dmB->labels               = listTmp;
186   depthTmp                  = dmA->depthLabel;
187   dmA->depthLabel           = dmB->depthLabel;
188   dmB->depthLabel           = depthTmp;
189   depthTmp                  = dmA->celltypeLabel;
190   dmA->celltypeLabel        = dmB->celltypeLabel;
191   dmB->celltypeLabel        = depthTmp;
192   tmpI                      = dmA->levelup;
193   dmA->levelup              = dmB->levelup;
194   dmB->levelup              = tmpI;
195   PetscFunctionReturn(PETSC_SUCCESS);
196 }
197 
198 PetscErrorCode DMPlexInterpolateInPlace_Internal(DM dm)
199 {
200   DM idm;
201 
202   PetscFunctionBegin;
203   PetscCall(DMPlexInterpolate(dm, &idm));
204   PetscCall(DMPlexCopyCoordinates(dm, idm));
205   PetscCall(DMPlexReplace_Internal(dm, &idm));
206   PetscFunctionReturn(PETSC_SUCCESS);
207 }
208 
209 /*@C
210   DMPlexCreateCoordinateSpace - Creates a finite element space for the coordinates
211 
212   Collective
213 
214   Input Parameters:
215 + dm        - The `DMPLEX`
216 . degree    - The degree of the finite element or `PETSC_DECIDE`
217 . project   - Flag to project current coordinates into the space
218 - coordFunc - An optional function to map new points from refinement to the surface
219 
220   Level: advanced
221 
222 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `PetscPointFunc`, `PetscFECreateLagrange()`, `DMGetCoordinateDM()`
223 @*/
224 PetscErrorCode DMPlexCreateCoordinateSpace(DM dm, PetscInt degree, PetscBool project, PetscPointFunc coordFunc)
225 {
226   DM_Plex *mesh = (DM_Plex *)dm->data;
227   PetscFE  fe   = NULL;
228   DM       cdm;
229   PetscInt dim, dE, qorder, height;
230 
231   PetscFunctionBegin;
232   PetscCall(DMGetDimension(dm, &dim));
233   PetscCall(DMGetCoordinateDim(dm, &dE));
234   qorder = degree;
235   PetscCall(DMGetCoordinateDM(dm, &cdm));
236   PetscObjectOptionsBegin((PetscObject)cdm);
237   PetscCall(PetscOptionsBoundedInt("-default_quadrature_order", "Quadrature order is one less than quadrature points per edge", "DMPlexCreateCoordinateSpace", qorder, &qorder, NULL, 0));
238   PetscOptionsEnd();
239   PetscCall(DMPlexGetVTKCellHeight(dm, &height));
240   if (degree >= 0) {
241     DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
242     PetscInt       cStart, cEnd, gct;
243 
244     PetscCall(DMPlexGetHeightStratum(dm, height, &cStart, &cEnd));
245     if (cEnd > cStart) PetscCall(DMPlexGetCellType(dm, cStart, &ct));
246     gct = (PetscInt)ct;
247     PetscCall(MPIU_Allreduce(MPI_IN_PLACE, &gct, 1, MPIU_INT, MPI_MIN, PetscObjectComm((PetscObject)dm)));
248     ct = (DMPolytopeType)gct;
249     // Work around current bug in PetscDualSpaceSetUp_Lagrange()
250     //   Can be seen in plex_tutorials-ex10_1
251     if (ct != DM_POLYTOPE_SEG_PRISM_TENSOR && ct != DM_POLYTOPE_TRI_PRISM_TENSOR && ct != DM_POLYTOPE_QUAD_PRISM_TENSOR) PetscCall(PetscFECreateLagrangeByCell(PETSC_COMM_SELF, dim, dE, ct, degree, qorder, &fe));
252   }
253   PetscCall(DMSetCoordinateDisc(dm, fe, project));
254   PetscCall(PetscFEDestroy(&fe));
255   mesh->coordFunc = coordFunc;
256   PetscFunctionReturn(PETSC_SUCCESS);
257 }
258 
259 /*@
260   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
261 
262   Collective
263 
264   Input Parameters:
265 + comm            - The communicator for the `DM` object
266 . dim             - The spatial dimension
267 . simplex         - Flag for simplicial cells, otherwise they are tensor product cells
268 . interpolate     - Flag to create intermediate mesh pieces (edges, faces)
269 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
270 
271   Output Parameter:
272 . newdm - The `DM` object
273 
274   Level: beginner
275 
276 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetType()`, `DMCreate()`
277 @*/
278 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscReal refinementLimit, DM *newdm)
279 {
280   DM          dm;
281   PetscMPIInt rank;
282 
283   PetscFunctionBegin;
284   PetscCall(DMCreate(comm, &dm));
285   PetscCall(DMSetType(dm, DMPLEX));
286   PetscCall(DMSetDimension(dm, dim));
287   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
288   PetscCallMPI(MPI_Comm_rank(comm, &rank));
289   switch (dim) {
290   case 2:
291     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "triangular"));
292     else PetscCall(PetscObjectSetName((PetscObject)dm, "quadrilateral"));
293     break;
294   case 3:
295     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "tetrahedral"));
296     else PetscCall(PetscObjectSetName((PetscObject)dm, "hexahedral"));
297     break;
298   default:
299     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
300   }
301   if (rank) {
302     PetscInt numPoints[2] = {0, 0};
303     PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL));
304   } else {
305     switch (dim) {
306     case 2:
307       if (simplex) {
308         PetscInt    numPoints[2]        = {4, 2};
309         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
310         PetscInt    cones[6]            = {2, 3, 4, 5, 4, 3};
311         PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
312         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
313 
314         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
315       } else {
316         PetscInt    numPoints[2]        = {6, 2};
317         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
318         PetscInt    cones[8]            = {2, 3, 4, 5, 3, 6, 7, 4};
319         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
320         PetscScalar vertexCoords[12]    = {-1.0, -0.5, 0.0, -0.5, 0.0, 0.5, -1.0, 0.5, 1.0, -0.5, 1.0, 0.5};
321 
322         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
323       }
324       break;
325     case 3:
326       if (simplex) {
327         PetscInt    numPoints[2]        = {5, 2};
328         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
329         PetscInt    cones[8]            = {4, 3, 5, 2, 5, 3, 4, 6};
330         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
331         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0};
332 
333         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
334       } else {
335         PetscInt    numPoints[2]         = {12, 2};
336         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
337         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9, 5, 4, 10, 11, 7, 12, 13, 8};
338         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
339         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5, -1.0, 0.5, -0.5, 0.0, 0.5, -0.5, 0.0, -0.5, -0.5, -1.0, -0.5, 0.5, 0.0, -0.5, 0.5, 0.0, 0.5, 0.5, -1.0, 0.5, 0.5, 1.0, 0.5, -0.5, 1.0, -0.5, -0.5, 1.0, -0.5, 0.5, 1.0, 0.5, 0.5};
340 
341         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
342       }
343       break;
344     default:
345       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
346     }
347   }
348   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
349   *newdm = dm;
350   if (refinementLimit > 0.0) {
351     DM          rdm;
352     const char *name;
353 
354     PetscCall(DMPlexSetRefinementUniform(*newdm, PETSC_FALSE));
355     PetscCall(DMPlexSetRefinementLimit(*newdm, refinementLimit));
356     PetscCall(DMRefine(*newdm, comm, &rdm));
357     PetscCall(PetscObjectGetName((PetscObject)*newdm, &name));
358     PetscCall(PetscObjectSetName((PetscObject)rdm, name));
359     PetscCall(DMDestroy(newdm));
360     *newdm = rdm;
361   }
362   if (interpolate) {
363     DM idm;
364 
365     PetscCall(DMPlexInterpolate(*newdm, &idm));
366     PetscCall(DMDestroy(newdm));
367     *newdm = idm;
368   }
369   PetscFunctionReturn(PETSC_SUCCESS);
370 }
371 
372 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
373 {
374   const PetscInt numVertices    = 2;
375   PetscInt       markerRight    = 1;
376   PetscInt       markerLeft     = 1;
377   PetscBool      markerSeparate = PETSC_FALSE;
378   Vec            coordinates;
379   PetscSection   coordSection;
380   PetscScalar   *coords;
381   PetscInt       coordSize;
382   PetscMPIInt    rank;
383   PetscInt       cdim = 1, v;
384 
385   PetscFunctionBegin;
386   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
387   if (markerSeparate) {
388     markerRight = 2;
389     markerLeft  = 1;
390   }
391   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
392   if (rank == 0) {
393     PetscCall(DMPlexSetChart(dm, 0, numVertices));
394     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
395     PetscCall(DMSetLabelValue(dm, "marker", 0, markerLeft));
396     PetscCall(DMSetLabelValue(dm, "marker", 1, markerRight));
397   }
398   PetscCall(DMPlexSymmetrize(dm));
399   PetscCall(DMPlexStratify(dm));
400   /* Build coordinates */
401   PetscCall(DMSetCoordinateDim(dm, cdim));
402   PetscCall(DMGetCoordinateSection(dm, &coordSection));
403   PetscCall(PetscSectionSetNumFields(coordSection, 1));
404   PetscCall(PetscSectionSetChart(coordSection, 0, numVertices));
405   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
406   for (v = 0; v < numVertices; ++v) {
407     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
408     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
409   }
410   PetscCall(PetscSectionSetUp(coordSection));
411   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
412   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
413   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
414   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
415   PetscCall(VecSetBlockSize(coordinates, cdim));
416   PetscCall(VecSetType(coordinates, VECSTANDARD));
417   PetscCall(VecGetArray(coordinates, &coords));
418   coords[0] = lower[0];
419   coords[1] = upper[0];
420   PetscCall(VecRestoreArray(coordinates, &coords));
421   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
422   PetscCall(VecDestroy(&coordinates));
423   PetscFunctionReturn(PETSC_SUCCESS);
424 }
425 
426 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
427 {
428   const PetscInt numVertices    = (edges[0] + 1) * (edges[1] + 1);
429   const PetscInt numEdges       = edges[0] * (edges[1] + 1) + (edges[0] + 1) * edges[1];
430   PetscInt       markerTop      = 1;
431   PetscInt       markerBottom   = 1;
432   PetscInt       markerRight    = 1;
433   PetscInt       markerLeft     = 1;
434   PetscBool      markerSeparate = PETSC_FALSE;
435   Vec            coordinates;
436   PetscSection   coordSection;
437   PetscScalar   *coords;
438   PetscInt       coordSize;
439   PetscMPIInt    rank;
440   PetscInt       v, vx, vy;
441 
442   PetscFunctionBegin;
443   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
444   if (markerSeparate) {
445     markerTop    = 3;
446     markerBottom = 1;
447     markerRight  = 2;
448     markerLeft   = 4;
449   }
450   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
451   if (rank == 0) {
452     PetscInt e, ex, ey;
453 
454     PetscCall(DMPlexSetChart(dm, 0, numEdges + numVertices));
455     for (e = 0; e < numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
456     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
457     for (vx = 0; vx <= edges[0]; vx++) {
458       for (ey = 0; ey < edges[1]; ey++) {
459         PetscInt edge   = vx * edges[1] + ey + edges[0] * (edges[1] + 1);
460         PetscInt vertex = ey * (edges[0] + 1) + vx + numEdges;
461         PetscInt cone[2];
462 
463         cone[0] = vertex;
464         cone[1] = vertex + edges[0] + 1;
465         PetscCall(DMPlexSetCone(dm, edge, cone));
466         if (vx == edges[0]) {
467           PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
468           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
469           if (ey == edges[1] - 1) {
470             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
471             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerRight));
472           }
473         } else if (vx == 0) {
474           PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
475           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
476           if (ey == edges[1] - 1) {
477             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
478             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft));
479           }
480         }
481       }
482     }
483     for (vy = 0; vy <= edges[1]; vy++) {
484       for (ex = 0; ex < edges[0]; ex++) {
485         PetscInt edge   = vy * edges[0] + ex;
486         PetscInt vertex = vy * (edges[0] + 1) + ex + numEdges;
487         PetscInt cone[2];
488 
489         cone[0] = vertex;
490         cone[1] = vertex + 1;
491         PetscCall(DMPlexSetCone(dm, edge, cone));
492         if (vy == edges[1]) {
493           PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
494           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
495           if (ex == edges[0] - 1) {
496             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
497             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerTop));
498           }
499         } else if (vy == 0) {
500           PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
501           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
502           if (ex == edges[0] - 1) {
503             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
504             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom));
505           }
506         }
507       }
508     }
509   }
510   PetscCall(DMPlexSymmetrize(dm));
511   PetscCall(DMPlexStratify(dm));
512   /* Build coordinates */
513   PetscCall(DMSetCoordinateDim(dm, 2));
514   PetscCall(DMGetCoordinateSection(dm, &coordSection));
515   PetscCall(PetscSectionSetNumFields(coordSection, 1));
516   PetscCall(PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices));
517   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 2));
518   for (v = numEdges; v < numEdges + numVertices; ++v) {
519     PetscCall(PetscSectionSetDof(coordSection, v, 2));
520     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 2));
521   }
522   PetscCall(PetscSectionSetUp(coordSection));
523   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
524   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
525   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
526   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
527   PetscCall(VecSetBlockSize(coordinates, 2));
528   PetscCall(VecSetType(coordinates, VECSTANDARD));
529   PetscCall(VecGetArray(coordinates, &coords));
530   for (vy = 0; vy <= edges[1]; ++vy) {
531     for (vx = 0; vx <= edges[0]; ++vx) {
532       coords[(vy * (edges[0] + 1) + vx) * 2 + 0] = lower[0] + ((upper[0] - lower[0]) / edges[0]) * vx;
533       coords[(vy * (edges[0] + 1) + vx) * 2 + 1] = lower[1] + ((upper[1] - lower[1]) / edges[1]) * vy;
534     }
535   }
536   PetscCall(VecRestoreArray(coordinates, &coords));
537   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
538   PetscCall(VecDestroy(&coordinates));
539   PetscFunctionReturn(PETSC_SUCCESS);
540 }
541 
542 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
543 {
544   PetscInt     vertices[3], numVertices;
545   PetscInt     numFaces       = 2 * faces[0] * faces[1] + 2 * faces[1] * faces[2] + 2 * faces[0] * faces[2];
546   PetscInt     markerTop      = 1;
547   PetscInt     markerBottom   = 1;
548   PetscInt     markerFront    = 1;
549   PetscInt     markerBack     = 1;
550   PetscInt     markerRight    = 1;
551   PetscInt     markerLeft     = 1;
552   PetscBool    markerSeparate = PETSC_FALSE;
553   Vec          coordinates;
554   PetscSection coordSection;
555   PetscScalar *coords;
556   PetscInt     coordSize;
557   PetscMPIInt  rank;
558   PetscInt     v, vx, vy, vz;
559   PetscInt     voffset, iface = 0, cone[4];
560 
561   PetscFunctionBegin;
562   PetscCheck(faces[0] >= 1 && faces[1] >= 1 && faces[2] >= 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
563   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
564   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
565   if (markerSeparate) {
566     markerBottom = 1;
567     markerTop    = 2;
568     markerFront  = 3;
569     markerBack   = 4;
570     markerRight  = 5;
571     markerLeft   = 6;
572   }
573   vertices[0] = faces[0] + 1;
574   vertices[1] = faces[1] + 1;
575   vertices[2] = faces[2] + 1;
576   numVertices = vertices[0] * vertices[1] * vertices[2];
577   if (rank == 0) {
578     PetscInt f;
579 
580     PetscCall(DMPlexSetChart(dm, 0, numFaces + numVertices));
581     for (f = 0; f < numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
582     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
583 
584     /* Side 0 (Top) */
585     for (vy = 0; vy < faces[1]; vy++) {
586       for (vx = 0; vx < faces[0]; vx++) {
587         voffset = numFaces + vertices[0] * vertices[1] * (vertices[2] - 1) + vy * vertices[0] + vx;
588         cone[0] = voffset;
589         cone[1] = voffset + 1;
590         cone[2] = voffset + vertices[0] + 1;
591         cone[3] = voffset + vertices[0];
592         PetscCall(DMPlexSetCone(dm, iface, cone));
593         PetscCall(DMSetLabelValue(dm, "marker", iface, markerTop));
594         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerTop));
595         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerTop));
596         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerTop));
597         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerTop));
598         iface++;
599       }
600     }
601 
602     /* Side 1 (Bottom) */
603     for (vy = 0; vy < faces[1]; vy++) {
604       for (vx = 0; vx < faces[0]; vx++) {
605         voffset = numFaces + vy * (faces[0] + 1) + vx;
606         cone[0] = voffset + 1;
607         cone[1] = voffset;
608         cone[2] = voffset + vertices[0];
609         cone[3] = voffset + vertices[0] + 1;
610         PetscCall(DMPlexSetCone(dm, iface, cone));
611         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBottom));
612         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBottom));
613         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBottom));
614         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerBottom));
615         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerBottom));
616         iface++;
617       }
618     }
619 
620     /* Side 2 (Front) */
621     for (vz = 0; vz < faces[2]; vz++) {
622       for (vx = 0; vx < faces[0]; vx++) {
623         voffset = numFaces + vz * vertices[0] * vertices[1] + vx;
624         cone[0] = voffset;
625         cone[1] = voffset + 1;
626         cone[2] = voffset + vertices[0] * vertices[1] + 1;
627         cone[3] = voffset + vertices[0] * vertices[1];
628         PetscCall(DMPlexSetCone(dm, iface, cone));
629         PetscCall(DMSetLabelValue(dm, "marker", iface, markerFront));
630         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerFront));
631         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerFront));
632         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerFront));
633         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerFront));
634         iface++;
635       }
636     }
637 
638     /* Side 3 (Back) */
639     for (vz = 0; vz < faces[2]; vz++) {
640       for (vx = 0; vx < faces[0]; vx++) {
641         voffset = numFaces + vz * vertices[0] * vertices[1] + vertices[0] * (vertices[1] - 1) + vx;
642         cone[0] = voffset + vertices[0] * vertices[1];
643         cone[1] = voffset + vertices[0] * vertices[1] + 1;
644         cone[2] = voffset + 1;
645         cone[3] = voffset;
646         PetscCall(DMPlexSetCone(dm, iface, cone));
647         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBack));
648         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBack));
649         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBack));
650         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerBack));
651         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerBack));
652         iface++;
653       }
654     }
655 
656     /* Side 4 (Left) */
657     for (vz = 0; vz < faces[2]; vz++) {
658       for (vy = 0; vy < faces[1]; vy++) {
659         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0];
660         cone[0] = voffset;
661         cone[1] = voffset + vertices[0] * vertices[1];
662         cone[2] = voffset + vertices[0] * vertices[1] + vertices[0];
663         cone[3] = voffset + vertices[0];
664         PetscCall(DMPlexSetCone(dm, iface, cone));
665         PetscCall(DMSetLabelValue(dm, "marker", iface, markerLeft));
666         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerLeft));
667         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerLeft));
668         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[1] + 0, markerLeft));
669         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerLeft));
670         iface++;
671       }
672     }
673 
674     /* Side 5 (Right) */
675     for (vz = 0; vz < faces[2]; vz++) {
676       for (vy = 0; vy < faces[1]; vy++) {
677         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0] + faces[0];
678         cone[0] = voffset + vertices[0] * vertices[1];
679         cone[1] = voffset;
680         cone[2] = voffset + vertices[0];
681         cone[3] = voffset + vertices[0] * vertices[1] + vertices[0];
682         PetscCall(DMPlexSetCone(dm, iface, cone));
683         PetscCall(DMSetLabelValue(dm, "marker", iface, markerRight));
684         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerRight));
685         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerRight));
686         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerRight));
687         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerRight));
688         iface++;
689       }
690     }
691   }
692   PetscCall(DMPlexSymmetrize(dm));
693   PetscCall(DMPlexStratify(dm));
694   /* Build coordinates */
695   PetscCall(DMSetCoordinateDim(dm, 3));
696   PetscCall(DMGetCoordinateSection(dm, &coordSection));
697   PetscCall(PetscSectionSetNumFields(coordSection, 1));
698   PetscCall(PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices));
699   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 3));
700   for (v = numFaces; v < numFaces + numVertices; ++v) {
701     PetscCall(PetscSectionSetDof(coordSection, v, 3));
702     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 3));
703   }
704   PetscCall(PetscSectionSetUp(coordSection));
705   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
706   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
707   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
708   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
709   PetscCall(VecSetBlockSize(coordinates, 3));
710   PetscCall(VecSetType(coordinates, VECSTANDARD));
711   PetscCall(VecGetArray(coordinates, &coords));
712   for (vz = 0; vz <= faces[2]; ++vz) {
713     for (vy = 0; vy <= faces[1]; ++vy) {
714       for (vx = 0; vx <= faces[0]; ++vx) {
715         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 0] = lower[0] + ((upper[0] - lower[0]) / faces[0]) * vx;
716         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 1] = lower[1] + ((upper[1] - lower[1]) / faces[1]) * vy;
717         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 2] = lower[2] + ((upper[2] - lower[2]) / faces[2]) * vz;
718       }
719     }
720   }
721   PetscCall(VecRestoreArray(coordinates, &coords));
722   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
723   PetscCall(VecDestroy(&coordinates));
724   PetscFunctionReturn(PETSC_SUCCESS);
725 }
726 
727 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate)
728 {
729   PetscFunctionBegin;
730   PetscValidLogicalCollectiveInt(dm, dim, 2);
731   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
732   PetscCall(DMSetDimension(dm, dim - 1));
733   PetscCall(DMSetCoordinateDim(dm, dim));
734   switch (dim) {
735   case 1:
736     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(dm, lower, upper, faces));
737     break;
738   case 2:
739     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(dm, lower, upper, faces));
740     break;
741   case 3:
742     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(dm, lower, upper, faces));
743     break;
744   default:
745     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Dimension not supported: %" PetscInt_FMT, dim);
746   }
747   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
748   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
749   PetscFunctionReturn(PETSC_SUCCESS);
750 }
751 
752 /*@C
753   DMPlexCreateBoxSurfaceMesh - Creates a mesh on the surface of the tensor product of unit intervals (box) using tensor cells (hexahedra).
754 
755   Collective
756 
757   Input Parameters:
758 + comm        - The communicator for the `DM` object
759 . dim         - The spatial dimension of the box, so the resulting mesh is has dimension `dim`-1
760 . faces       - Number of faces per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
761 . lower       - The lower left corner, or `NULL` for (0, 0, 0)
762 . upper       - The upper right corner, or `NULL` for (1, 1, 1)
763 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
764 
765   Output Parameter:
766 . dm - The `DM` object
767 
768   Level: beginner
769 
770 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateBoxMesh()`, `DMPlexCreateFromFile()`, `DMSetType()`, `DMCreate()`
771 @*/
772 PetscErrorCode DMPlexCreateBoxSurfaceMesh(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate, DM *dm)
773 {
774   PetscInt  fac[3] = {1, 1, 1};
775   PetscReal low[3] = {0, 0, 0};
776   PetscReal upp[3] = {1, 1, 1};
777 
778   PetscFunctionBegin;
779   PetscCall(DMCreate(comm, dm));
780   PetscCall(DMSetType(*dm, DMPLEX));
781   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(*dm, dim, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, interpolate));
782   PetscFunctionReturn(PETSC_SUCCESS);
783 }
784 
785 static PetscErrorCode DMPlexCreateLineMesh_Internal(DM dm, PetscInt segments, PetscReal lower, PetscReal upper, DMBoundaryType bd)
786 {
787   PetscInt     i, fStart, fEnd, numCells = 0, numVerts = 0;
788   PetscInt     numPoints[2], *coneSize, *cones, *coneOrientations;
789   PetscScalar *vertexCoords;
790   PetscReal    L, maxCell;
791   PetscBool    markerSeparate = PETSC_FALSE;
792   PetscInt     markerLeft = 1, faceMarkerLeft = 1;
793   PetscInt     markerRight = 1, faceMarkerRight = 2;
794   PetscBool    wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
795   PetscMPIInt  rank;
796 
797   PetscFunctionBegin;
798   PetscAssertPointer(dm, 1);
799 
800   PetscCall(DMSetDimension(dm, 1));
801   PetscCall(DMCreateLabel(dm, "marker"));
802   PetscCall(DMCreateLabel(dm, "Face Sets"));
803 
804   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
805   if (rank == 0) numCells = segments;
806   if (rank == 0) numVerts = segments + (wrap ? 0 : 1);
807 
808   numPoints[0] = numVerts;
809   numPoints[1] = numCells;
810   PetscCall(PetscMalloc4(numCells + numVerts, &coneSize, numCells * 2, &cones, numCells + numVerts, &coneOrientations, numVerts, &vertexCoords));
811   PetscCall(PetscArrayzero(coneOrientations, numCells + numVerts));
812   for (i = 0; i < numCells; ++i) coneSize[i] = 2;
813   for (i = 0; i < numVerts; ++i) coneSize[numCells + i] = 0;
814   for (i = 0; i < numCells; ++i) {
815     cones[2 * i]     = numCells + i % numVerts;
816     cones[2 * i + 1] = numCells + (i + 1) % numVerts;
817   }
818   for (i = 0; i < numVerts; ++i) vertexCoords[i] = lower + (upper - lower) * ((PetscReal)i / (PetscReal)numCells);
819   PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
820   PetscCall(PetscFree4(coneSize, cones, coneOrientations, vertexCoords));
821 
822   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
823   if (markerSeparate) {
824     markerLeft  = faceMarkerLeft;
825     markerRight = faceMarkerRight;
826   }
827   if (!wrap && rank == 0) {
828     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
829     PetscCall(DMSetLabelValue(dm, "marker", fStart, markerLeft));
830     PetscCall(DMSetLabelValue(dm, "marker", fEnd - 1, markerRight));
831     PetscCall(DMSetLabelValue(dm, "Face Sets", fStart, faceMarkerLeft));
832     PetscCall(DMSetLabelValue(dm, "Face Sets", fEnd - 1, faceMarkerRight));
833   }
834   if (wrap) {
835     L       = upper - lower;
836     maxCell = (PetscReal)1.1 * (L / (PetscReal)PetscMax(1, segments));
837     PetscCall(DMSetPeriodicity(dm, &maxCell, &lower, &L));
838   }
839   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
840   PetscFunctionReturn(PETSC_SUCCESS);
841 }
842 
843 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
844 {
845   DM      boundary, vol;
846   DMLabel bdlabel;
847 
848   PetscFunctionBegin;
849   PetscAssertPointer(dm, 1);
850   for (PetscInt i = 0; i < dim; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
851   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &boundary));
852   PetscCall(DMSetType(boundary, DMPLEX));
853   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(boundary, dim, faces, lower, upper, PETSC_FALSE));
854   PetscCall(DMPlexGenerate(boundary, NULL, interpolate, &vol));
855   PetscCall(DMGetLabel(vol, "marker", &bdlabel));
856   if (bdlabel) PetscCall(DMPlexLabelComplete(vol, bdlabel));
857   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_FALSE, vol));
858   PetscCall(DMPlexReplace_Internal(dm, &vol));
859   PetscCall(DMDestroy(&boundary));
860   PetscFunctionReturn(PETSC_SUCCESS);
861 }
862 
863 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
864 {
865   DMLabel     cutLabel  = NULL;
866   PetscInt    markerTop = 1, faceMarkerTop = 1;
867   PetscInt    markerBottom = 1, faceMarkerBottom = 1;
868   PetscInt    markerFront = 1, faceMarkerFront = 1;
869   PetscInt    markerBack = 1, faceMarkerBack = 1;
870   PetscInt    markerRight = 1, faceMarkerRight = 1;
871   PetscInt    markerLeft = 1, faceMarkerLeft = 1;
872   PetscInt    dim;
873   PetscBool   markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
874   PetscMPIInt rank;
875 
876   PetscFunctionBegin;
877   PetscCall(DMGetDimension(dm, &dim));
878   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
879   PetscCall(DMCreateLabel(dm, "marker"));
880   PetscCall(DMCreateLabel(dm, "Face Sets"));
881   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
882   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST || bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST || bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
883     if (cutMarker) {
884       PetscCall(DMCreateLabel(dm, "periodic_cut"));
885       PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
886     }
887   }
888   switch (dim) {
889   case 2:
890     faceMarkerTop    = 3;
891     faceMarkerBottom = 1;
892     faceMarkerRight  = 2;
893     faceMarkerLeft   = 4;
894     break;
895   case 3:
896     faceMarkerBottom = 1;
897     faceMarkerTop    = 2;
898     faceMarkerFront  = 3;
899     faceMarkerBack   = 4;
900     faceMarkerRight  = 5;
901     faceMarkerLeft   = 6;
902     break;
903   default:
904     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
905   }
906   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
907   if (markerSeparate) {
908     markerBottom = faceMarkerBottom;
909     markerTop    = faceMarkerTop;
910     markerFront  = faceMarkerFront;
911     markerBack   = faceMarkerBack;
912     markerRight  = faceMarkerRight;
913     markerLeft   = faceMarkerLeft;
914   }
915   {
916     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
917     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
918     const PetscInt numZEdges    = rank == 0 ? edges[2] : 0;
919     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0] + 1) : 0;
920     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1] + 1) : 0;
921     const PetscInt numZVertices = rank == 0 ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2] + 1) : 0;
922     const PetscInt numCells     = numXEdges * numYEdges * numZEdges;
923     const PetscInt numXFaces    = numYEdges * numZEdges;
924     const PetscInt numYFaces    = numXEdges * numZEdges;
925     const PetscInt numZFaces    = numXEdges * numYEdges;
926     const PetscInt numTotXFaces = numXVertices * numXFaces;
927     const PetscInt numTotYFaces = numYVertices * numYFaces;
928     const PetscInt numTotZFaces = numZVertices * numZFaces;
929     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
930     const PetscInt numTotXEdges = numXEdges * numYVertices * numZVertices;
931     const PetscInt numTotYEdges = numYEdges * numXVertices * numZVertices;
932     const PetscInt numTotZEdges = numZEdges * numXVertices * numYVertices;
933     const PetscInt numVertices  = numXVertices * numYVertices * numZVertices;
934     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
935     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
936     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
937     const PetscInt firstYFace   = firstXFace + numTotXFaces;
938     const PetscInt firstZFace   = firstYFace + numTotYFaces;
939     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
940     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
941     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
942     Vec            coordinates;
943     PetscSection   coordSection;
944     PetscScalar   *coords;
945     PetscInt       coordSize;
946     PetscInt       v, vx, vy, vz;
947     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
948 
949     PetscCall(DMPlexSetChart(dm, 0, numCells + numFaces + numEdges + numVertices));
950     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
951     for (f = firstXFace; f < firstXFace + numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
952     for (e = firstXEdge; e < firstXEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
953     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
954     /* Build cells */
955     for (fz = 0; fz < numZEdges; ++fz) {
956       for (fy = 0; fy < numYEdges; ++fy) {
957         for (fx = 0; fx < numXEdges; ++fx) {
958           PetscInt cell  = (fz * numYEdges + fy) * numXEdges + fx;
959           PetscInt faceB = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
960           PetscInt faceT = firstZFace + (fy * numXEdges + fx) * numZVertices + ((fz + 1) % numZVertices);
961           PetscInt faceF = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
962           PetscInt faceK = firstYFace + (fz * numXEdges + fx) * numYVertices + ((fy + 1) % numYVertices);
963           PetscInt faceL = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
964           PetscInt faceR = firstXFace + (fz * numYEdges + fy) * numXVertices + ((fx + 1) % numXVertices);
965           /* B,  T,  F,  K,  R,  L */
966           PetscInt ornt[6] = {-2, 0, 0, -3, 0, -2}; /* ??? */
967           PetscInt cone[6];
968 
969           /* no boundary twisting in 3D */
970           cone[0] = faceB;
971           cone[1] = faceT;
972           cone[2] = faceF;
973           cone[3] = faceK;
974           cone[4] = faceR;
975           cone[5] = faceL;
976           PetscCall(DMPlexSetCone(dm, cell, cone));
977           PetscCall(DMPlexSetConeOrientation(dm, cell, ornt));
978           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
979           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
980           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
981         }
982       }
983     }
984     /* Build x faces */
985     for (fz = 0; fz < numZEdges; ++fz) {
986       for (fy = 0; fy < numYEdges; ++fy) {
987         for (fx = 0; fx < numXVertices; ++fx) {
988           PetscInt face    = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
989           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
990           PetscInt edgeR   = firstZEdge + (((fy + 1) % numYVertices) * numXVertices + fx) * numZEdges + fz;
991           PetscInt edgeB   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
992           PetscInt edgeT   = firstYEdge + (((fz + 1) % numZVertices) * numXVertices + fx) * numYEdges + fy;
993           PetscInt ornt[4] = {0, 0, -1, -1};
994           PetscInt cone[4];
995 
996           if (dim == 3) {
997             /* markers */
998             if (bdX != DM_BOUNDARY_PERIODIC) {
999               if (fx == numXVertices - 1) {
1000                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight));
1001                 PetscCall(DMSetLabelValue(dm, "marker", face, markerRight));
1002               } else if (fx == 0) {
1003                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft));
1004                 PetscCall(DMSetLabelValue(dm, "marker", face, markerLeft));
1005               }
1006             }
1007           }
1008           cone[0] = edgeB;
1009           cone[1] = edgeR;
1010           cone[2] = edgeT;
1011           cone[3] = edgeL;
1012           PetscCall(DMPlexSetCone(dm, face, cone));
1013           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1014         }
1015       }
1016     }
1017     /* Build y faces */
1018     for (fz = 0; fz < numZEdges; ++fz) {
1019       for (fx = 0; fx < numXEdges; ++fx) {
1020         for (fy = 0; fy < numYVertices; ++fy) {
1021           PetscInt face    = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
1022           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
1023           PetscInt edgeR   = firstZEdge + (fy * numXVertices + ((fx + 1) % numXVertices)) * numZEdges + fz;
1024           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1025           PetscInt edgeT   = firstXEdge + (((fz + 1) % numZVertices) * numYVertices + fy) * numXEdges + fx;
1026           PetscInt ornt[4] = {0, 0, -1, -1};
1027           PetscInt cone[4];
1028 
1029           if (dim == 3) {
1030             /* markers */
1031             if (bdY != DM_BOUNDARY_PERIODIC) {
1032               if (fy == numYVertices - 1) {
1033                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack));
1034                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBack));
1035               } else if (fy == 0) {
1036                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront));
1037                 PetscCall(DMSetLabelValue(dm, "marker", face, markerFront));
1038               }
1039             }
1040           }
1041           cone[0] = edgeB;
1042           cone[1] = edgeR;
1043           cone[2] = edgeT;
1044           cone[3] = edgeL;
1045           PetscCall(DMPlexSetCone(dm, face, cone));
1046           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1047         }
1048       }
1049     }
1050     /* Build z faces */
1051     for (fy = 0; fy < numYEdges; ++fy) {
1052       for (fx = 0; fx < numXEdges; ++fx) {
1053         for (fz = 0; fz < numZVertices; fz++) {
1054           PetscInt face    = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
1055           PetscInt edgeL   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
1056           PetscInt edgeR   = firstYEdge + (fz * numXVertices + ((fx + 1) % numXVertices)) * numYEdges + fy;
1057           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1058           PetscInt edgeT   = firstXEdge + (fz * numYVertices + ((fy + 1) % numYVertices)) * numXEdges + fx;
1059           PetscInt ornt[4] = {0, 0, -1, -1};
1060           PetscInt cone[4];
1061 
1062           if (dim == 2) {
1063             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges - 1) {
1064               edgeR += numYEdges - 1 - 2 * fy;
1065               ornt[1] = -1;
1066             }
1067             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges - 1) {
1068               edgeT += numXEdges - 1 - 2 * fx;
1069               ornt[2] = 0;
1070             }
1071             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1072             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1073           } else {
1074             /* markers */
1075             if (bdZ != DM_BOUNDARY_PERIODIC) {
1076               if (fz == numZVertices - 1) {
1077                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop));
1078                 PetscCall(DMSetLabelValue(dm, "marker", face, markerTop));
1079               } else if (fz == 0) {
1080                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom));
1081                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBottom));
1082               }
1083             }
1084           }
1085           cone[0] = edgeB;
1086           cone[1] = edgeR;
1087           cone[2] = edgeT;
1088           cone[3] = edgeL;
1089           PetscCall(DMPlexSetCone(dm, face, cone));
1090           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1091         }
1092       }
1093     }
1094     /* Build Z edges*/
1095     for (vy = 0; vy < numYVertices; vy++) {
1096       for (vx = 0; vx < numXVertices; vx++) {
1097         for (ez = 0; ez < numZEdges; ez++) {
1098           const PetscInt edge    = firstZEdge + (vy * numXVertices + vx) * numZEdges + ez;
1099           const PetscInt vertexB = firstVertex + (ez * numYVertices + vy) * numXVertices + vx;
1100           const PetscInt vertexT = firstVertex + (((ez + 1) % numZVertices) * numYVertices + vy) * numXVertices + vx;
1101           PetscInt       cone[2];
1102 
1103           cone[0] = vertexB;
1104           cone[1] = vertexT;
1105           PetscCall(DMPlexSetCone(dm, edge, cone));
1106           if (dim == 3) {
1107             if (bdX != DM_BOUNDARY_PERIODIC) {
1108               if (vx == numXVertices - 1) {
1109                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1110                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1111                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1112               } else if (vx == 0) {
1113                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1114                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1115                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1116               }
1117             }
1118             if (bdY != DM_BOUNDARY_PERIODIC) {
1119               if (vy == numYVertices - 1) {
1120                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1121                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1122                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1123               } else if (vy == 0) {
1124                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1125                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1126                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1127               }
1128             }
1129           }
1130         }
1131       }
1132     }
1133     /* Build Y edges*/
1134     for (vz = 0; vz < numZVertices; vz++) {
1135       for (vx = 0; vx < numXVertices; vx++) {
1136         for (ey = 0; ey < numYEdges; ey++) {
1137           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges - 1) ? (numXVertices - vx - 1) : (vz * numYVertices + ((ey + 1) % numYVertices)) * numXVertices + vx;
1138           const PetscInt edge    = firstYEdge + (vz * numXVertices + vx) * numYEdges + ey;
1139           const PetscInt vertexF = firstVertex + (vz * numYVertices + ey) * numXVertices + vx;
1140           const PetscInt vertexK = firstVertex + nextv;
1141           PetscInt       cone[2];
1142 
1143           cone[0] = vertexF;
1144           cone[1] = vertexK;
1145           PetscCall(DMPlexSetCone(dm, edge, cone));
1146           if (dim == 2) {
1147             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1148               if (vx == numXVertices - 1) {
1149                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight));
1150                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1151                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1152                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1153               } else if (vx == 0) {
1154                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft));
1155                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1156                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1157                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1158               }
1159             } else {
1160               if (vx == 0 && cutLabel) {
1161                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1162                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1163                 if (ey == numYEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1164               }
1165             }
1166           } else {
1167             if (bdX != DM_BOUNDARY_PERIODIC) {
1168               if (vx == numXVertices - 1) {
1169                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1170                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1171                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1172               } else if (vx == 0) {
1173                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1174                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1175                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1176               }
1177             }
1178             if (bdZ != DM_BOUNDARY_PERIODIC) {
1179               if (vz == numZVertices - 1) {
1180                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1181                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1182                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1183               } else if (vz == 0) {
1184                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1185                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1186                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1187               }
1188             }
1189           }
1190         }
1191       }
1192     }
1193     /* Build X edges*/
1194     for (vz = 0; vz < numZVertices; vz++) {
1195       for (vy = 0; vy < numYVertices; vy++) {
1196         for (ex = 0; ex < numXEdges; ex++) {
1197           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges - 1) ? (numYVertices - vy - 1) * numXVertices : (vz * numYVertices + vy) * numXVertices + (ex + 1) % numXVertices;
1198           const PetscInt edge    = firstXEdge + (vz * numYVertices + vy) * numXEdges + ex;
1199           const PetscInt vertexL = firstVertex + (vz * numYVertices + vy) * numXVertices + ex;
1200           const PetscInt vertexR = firstVertex + nextv;
1201           PetscInt       cone[2];
1202 
1203           cone[0] = vertexL;
1204           cone[1] = vertexR;
1205           PetscCall(DMPlexSetCone(dm, edge, cone));
1206           if (dim == 2) {
1207             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1208               if (vy == numYVertices - 1) {
1209                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop));
1210                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1211                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1212                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1213               } else if (vy == 0) {
1214                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom));
1215                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1216                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1217                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1218               }
1219             } else {
1220               if (vy == 0 && cutLabel) {
1221                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1222                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1223                 if (ex == numXEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1224               }
1225             }
1226           } else {
1227             if (bdY != DM_BOUNDARY_PERIODIC) {
1228               if (vy == numYVertices - 1) {
1229                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1230                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1231                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1232               } else if (vy == 0) {
1233                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1234                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1235                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1236               }
1237             }
1238             if (bdZ != DM_BOUNDARY_PERIODIC) {
1239               if (vz == numZVertices - 1) {
1240                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1241                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1242                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1243               } else if (vz == 0) {
1244                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1245                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1246                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1247               }
1248             }
1249           }
1250         }
1251       }
1252     }
1253     PetscCall(DMPlexSymmetrize(dm));
1254     PetscCall(DMPlexStratify(dm));
1255     /* Build coordinates */
1256     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1257     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1258     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1259     PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVertices));
1260     for (v = firstVertex; v < firstVertex + numVertices; ++v) {
1261       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1262       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1263     }
1264     PetscCall(PetscSectionSetUp(coordSection));
1265     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1266     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1267     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1268     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1269     PetscCall(VecSetBlockSize(coordinates, dim));
1270     PetscCall(VecSetType(coordinates, VECSTANDARD));
1271     PetscCall(VecGetArray(coordinates, &coords));
1272     for (vz = 0; vz < numZVertices; ++vz) {
1273       for (vy = 0; vy < numYVertices; ++vy) {
1274         for (vx = 0; vx < numXVertices; ++vx) {
1275           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * vx;
1276           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * vy;
1277           if (dim == 3) coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 2] = lower[2] + ((upper[2] - lower[2]) / numZEdges) * vz;
1278         }
1279       }
1280     }
1281     PetscCall(VecRestoreArray(coordinates, &coords));
1282     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1283     PetscCall(VecDestroy(&coordinates));
1284   }
1285   PetscFunctionReturn(PETSC_SUCCESS);
1286 }
1287 
1288 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1289 {
1290   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1291   PetscInt       fac[3] = {0, 0, 0}, d;
1292 
1293   PetscFunctionBegin;
1294   PetscAssertPointer(dm, 1);
1295   PetscValidLogicalCollectiveInt(dm, dim, 2);
1296   PetscCall(DMSetDimension(dm, dim));
1297   for (d = 0; d < dim; ++d) {
1298     fac[d] = faces[d];
1299     bdt[d] = periodicity[d];
1300   }
1301   PetscCall(DMPlexCreateCubeMesh_Internal(dm, lower, upper, fac, bdt[0], bdt[1], bdt[2]));
1302   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST || periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST || (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
1303     PetscReal L[3]       = {-1., -1., 0.};
1304     PetscReal maxCell[3] = {-1., -1., 0.};
1305 
1306     for (d = 0; d < dim; ++d) {
1307       if (periodicity[d] != DM_BOUNDARY_NONE) {
1308         L[d]       = upper[d] - lower[d];
1309         maxCell[d] = 1.1 * (L[d] / PetscMax(1, faces[d]));
1310       }
1311     }
1312     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1313   }
1314   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
1315   PetscFunctionReturn(PETSC_SUCCESS);
1316 }
1317 
1318 static PetscErrorCode DMPlexCreateBoxMesh_Internal(DM dm, DMPlexShape shape, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1319 {
1320   PetscFunctionBegin;
1321   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
1322   if (shape == DM_SHAPE_ZBOX) PetscCall(DMPlexCreateBoxMesh_Tensor_SFC_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1323   else if (dim == 1) PetscCall(DMPlexCreateLineMesh_Internal(dm, faces[0], lower[0], upper[0], periodicity[0]));
1324   else if (simplex) PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1325   else PetscCall(DMPlexCreateBoxMesh_Tensor_Internal(dm, dim, faces, lower, upper, periodicity));
1326   if (!interpolate && dim > 1 && !simplex) {
1327     DM udm;
1328 
1329     PetscCall(DMPlexUninterpolate(dm, &udm));
1330     PetscCall(DMPlexCopyCoordinates(dm, udm));
1331     PetscCall(DMPlexReplace_Internal(dm, &udm));
1332   }
1333   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
1334   PetscFunctionReturn(PETSC_SUCCESS);
1335 }
1336 
1337 /*@C
1338   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
1339 
1340   Collective
1341 
1342   Input Parameters:
1343 + comm        - The communicator for the `DM` object
1344 . dim         - The spatial dimension
1345 . simplex     - `PETSC_TRUE` for simplices, `PETSC_FALSE` for tensor cells
1346 . faces       - Number of faces per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1347 . lower       - The lower left corner, or `NULL` for (0, 0, 0)
1348 . upper       - The upper right corner, or `NULL` for (1, 1, 1)
1349 . periodicity - The boundary type for the X,Y,Z direction, or `NULL` for `DM_BOUNDARY_NONE`
1350 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1351 
1352   Output Parameter:
1353 . dm - The `DM` object
1354 
1355   Level: beginner
1356 
1357   Note:
1358   To customize this mesh using options, use
1359 .vb
1360   DMCreate(comm, &dm);
1361   DMSetType(dm, DMPLEX);
1362   DMSetFromOptions(dm);
1363 .ve
1364   and use the options in `DMSetFromOptions()`.
1365 
1366   Here is the numbering returned for 2 faces in each direction for tensor cells\:
1367 .vb
1368  10---17---11---18----12
1369   |         |         |
1370   |         |         |
1371  20    2   22    3    24
1372   |         |         |
1373   |         |         |
1374   7---15----8---16----9
1375   |         |         |
1376   |         |         |
1377  19    0   21    1   23
1378   |         |         |
1379   |         |         |
1380   4---13----5---14----6
1381 .ve
1382   and for simplicial cells
1383 .vb
1384  14----8---15----9----16
1385   |\     5  |\      7 |
1386   | \       | \       |
1387  13   2    14    3    15
1388   | 4   \   | 6   \   |
1389   |       \ |       \ |
1390  11----6---12----7----13
1391   |\        |\        |
1392   | \    1  | \     3 |
1393  10   0    11    1    12
1394   | 0   \   | 2   \   |
1395   |       \ |       \ |
1396   8----4----9----5----10
1397 .ve
1398 
1399 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
1400 @*/
1401 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
1402 {
1403   PetscInt       fac[3] = {1, 1, 1};
1404   PetscReal      low[3] = {0, 0, 0};
1405   PetscReal      upp[3] = {1, 1, 1};
1406   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1407 
1408   PetscFunctionBegin;
1409   PetscCall(DMCreate(comm, dm));
1410   PetscCall(DMSetType(*dm, DMPLEX));
1411   PetscCall(DMPlexCreateBoxMesh_Internal(*dm, DM_SHAPE_BOX, dim, simplex, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt, interpolate));
1412   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1413   PetscFunctionReturn(PETSC_SUCCESS);
1414 }
1415 
1416 static PetscErrorCode DMPlexCreateWedgeBoxMesh_Internal(DM dm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1417 {
1418   DM       bdm, vol;
1419   PetscInt i;
1420 
1421   PetscFunctionBegin;
1422   // TODO Now we can support periodicity
1423   for (i = 0; i < 3; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity not yet supported");
1424   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &bdm));
1425   PetscCall(DMSetType(bdm, DMPLEX));
1426   PetscCall(DMSetDimension(bdm, 2));
1427   PetscCall(PetscLogEventBegin(DMPLEX_Generate, bdm, 0, 0, 0));
1428   PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(bdm, 2, faces, lower, upper, periodicity, PETSC_TRUE));
1429   PetscCall(DMPlexExtrude(bdm, faces[2], upper[2] - lower[2], PETSC_TRUE, PETSC_FALSE, PETSC_FALSE, NULL, NULL, &vol));
1430   PetscCall(PetscLogEventEnd(DMPLEX_Generate, bdm, 0, 0, 0));
1431   PetscCall(DMDestroy(&bdm));
1432   PetscCall(DMPlexReplace_Internal(dm, &vol));
1433   if (lower[2] != 0.0) {
1434     Vec          v;
1435     PetscScalar *x;
1436     PetscInt     cDim, n;
1437 
1438     PetscCall(DMGetCoordinatesLocal(dm, &v));
1439     PetscCall(VecGetBlockSize(v, &cDim));
1440     PetscCall(VecGetLocalSize(v, &n));
1441     PetscCall(VecGetArray(v, &x));
1442     x += cDim;
1443     for (i = 0; i < n; i += cDim) x[i] += lower[2];
1444     PetscCall(VecRestoreArray(v, &x));
1445     PetscCall(DMSetCoordinatesLocal(dm, v));
1446   }
1447   PetscFunctionReturn(PETSC_SUCCESS);
1448 }
1449 
1450 /*@
1451   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tessellating the (x,y) plane and extruding in the third direction using wedge cells.
1452 
1453   Collective
1454 
1455   Input Parameters:
1456 + comm        - The communicator for the `DM` object
1457 . faces       - Number of faces per dimension, or `NULL` for (1, 1, 1)
1458 . lower       - The lower left corner, or `NULL` for (0, 0, 0)
1459 . upper       - The upper right corner, or `NULL` for (1, 1, 1)
1460 . periodicity - The boundary type for the X,Y,Z direction, or `NULL` for `DM_BOUNDARY_NONE`
1461 . orderHeight - If `PETSC_TRUE`, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1462 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1463 
1464   Output Parameter:
1465 . dm - The `DM` object
1466 
1467   Level: beginner
1468 
1469 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateWedgeCylinderMesh()`, `DMExtrude()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1470 @*/
1471 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool orderHeight, PetscBool interpolate, DM *dm)
1472 {
1473   PetscInt       fac[3] = {1, 1, 1};
1474   PetscReal      low[3] = {0, 0, 0};
1475   PetscReal      upp[3] = {1, 1, 1};
1476   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1477 
1478   PetscFunctionBegin;
1479   PetscCall(DMCreate(comm, dm));
1480   PetscCall(DMSetType(*dm, DMPLEX));
1481   PetscCall(DMPlexCreateWedgeBoxMesh_Internal(*dm, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt));
1482   if (!interpolate) {
1483     DM udm;
1484 
1485     PetscCall(DMPlexUninterpolate(*dm, &udm));
1486     PetscCall(DMPlexReplace_Internal(*dm, &udm));
1487   }
1488   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1489   PetscFunctionReturn(PETSC_SUCCESS);
1490 }
1491 
1492 /*
1493   DMPlexTensorPointLexicographic_Private - Returns all tuples of size 'len' with nonnegative integers that are all less than or equal to 'max' for that dimension.
1494 
1495   Input Parameters:
1496 + len - The length of the tuple
1497 . max - The maximum for each dimension, so values are in [0, max)
1498 - tup - A tuple of length len+1: tup[len] > 0 indicates a stopping condition
1499 
1500   Output Parameter:
1501 . tup - A tuple of `len` integers whose entries are at most `max`
1502 
1503   Level: developer
1504 
1505   Note:
1506   Ordering is lexicographic with lowest index as least significant in ordering.
1507   e.g. for len == 2 and max == 2, this will return, in order, {0,0}, {1,0}, {2,0}, {0,1}, {1,1}, {2,1}, {0,2}, {1,2}, {2,2}.
1508 
1509 .seealso: PetscDualSpaceTensorPointLexicographic_Internal(), PetscDualSpaceLatticePointLexicographic_Internal()
1510 */
1511 static PetscErrorCode DMPlexTensorPointLexicographic_Private(PetscInt len, const PetscInt max[], PetscInt tup[])
1512 {
1513   PetscInt i;
1514 
1515   PetscFunctionBegin;
1516   for (i = 0; i < len; ++i) {
1517     if (tup[i] < max[i] - 1) {
1518       break;
1519     } else {
1520       tup[i] = 0;
1521     }
1522   }
1523   if (i == len) tup[i - 1] = max[i - 1];
1524   else ++tup[i];
1525   PetscFunctionReturn(PETSC_SUCCESS);
1526 }
1527 
1528 static PetscInt TupleToIndex_Private(PetscInt len, const PetscInt max[], const PetscInt tup[])
1529 {
1530   PetscInt i, idx = tup[len - 1];
1531 
1532   for (i = len - 2; i >= 0; --i) {
1533     idx *= max[i];
1534     idx += tup[i];
1535   }
1536   return idx;
1537 }
1538 
1539 static PetscErrorCode DestroyExtent_Private(void *extent)
1540 {
1541   return PetscFree(extent);
1542 }
1543 
1544 static PetscErrorCode DMPlexCreateHypercubicMesh_Internal(DM dm, PetscInt dim, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], const DMBoundaryType bd[])
1545 {
1546   Vec          coordinates;
1547   PetscSection coordSection;
1548   DMLabel      cutLabel    = NULL;
1549   PetscBool    cutMarker   = PETSC_FALSE;
1550   PetscBool    periodic    = PETSC_FALSE;
1551   PetscInt     numCells    = 1, c;
1552   PetscInt     numVertices = 1, v;
1553   PetscScalar *coords;
1554   PetscInt    *vertices, *vert, *vtmp, *supp, cone[2];
1555   PetscInt     d, e, cell = 0, coordSize;
1556   PetscMPIInt  rank;
1557 
1558   PetscFunctionBegin;
1559   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1560   PetscCall(DMSetDimension(dm, dim));
1561   PetscCall(PetscCalloc4(dim, &vertices, dim, &vert, dim, &vtmp, 2 * dim, &supp));
1562   PetscCall(DMCreateLabel(dm, "marker"));
1563   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
1564   for (d = 0; d < dim; ++d) periodic = (periodic || bd[d] == DM_BOUNDARY_PERIODIC) ? PETSC_TRUE : PETSC_FALSE;
1565   if (periodic && cutMarker) {
1566     PetscCall(DMCreateLabel(dm, "periodic_cut"));
1567     PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
1568   }
1569   for (d = 0; d < dim; ++d) PetscCheck(bd[d] == DM_BOUNDARY_PERIODIC, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Hypercubic mesh must be periodic now");
1570   for (d = 0; d < dim; ++d) {
1571     vertices[d] = edges[d];
1572     numVertices *= vertices[d];
1573   }
1574   numCells = numVertices * dim;
1575   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1576   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, 2));
1577   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetSupportSize(dm, v, 2 * dim));
1578   /* TODO Loop over boundary and reset support sizes */
1579   PetscCall(DMSetUp(dm)); /* Allocate space for cones and supports */
1580   /* Build cell cones and vertex supports */
1581   PetscCall(DMCreateLabel(dm, "celltype"));
1582   while (vert[dim - 1] < vertices[dim - 1]) {
1583     const PetscInt vertex = TupleToIndex_Private(dim, vertices, vert) + numCells;
1584     PetscInt       s      = 0;
1585 
1586     PetscCall(PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT ":", vertex));
1587     for (d = 0; d < dim; ++d) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, vert[d]));
1588     PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
1589     PetscCall(DMPlexSetCellType(dm, vertex, DM_POLYTOPE_POINT));
1590     for (d = 0; d < dim; ++d) {
1591       for (e = 0; e < dim; ++e) vtmp[e] = vert[e];
1592       vtmp[d] = (vert[d] + 1) % vertices[d];
1593       cone[0] = vertex;
1594       cone[1] = TupleToIndex_Private(dim, vertices, vtmp) + numCells;
1595       PetscCall(PetscPrintf(PETSC_COMM_SELF, "  Vertex %" PetscInt_FMT ":", cone[1]));
1596       for (e = 0; e < dim; ++e) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, vtmp[e]));
1597       PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
1598       PetscCall(DMPlexSetCone(dm, cell, cone));
1599       PetscCall(DMPlexSetCellType(dm, cell, DM_POLYTOPE_SEGMENT));
1600       PetscCall(PetscPrintf(PETSC_COMM_SELF, "  Edge %" PetscInt_FMT " (%" PetscInt_FMT " %" PetscInt_FMT ")\n", cell, cone[0], cone[1]));
1601       ++cell;
1602     }
1603     for (d = 0; d < dim; ++d) {
1604       for (e = 0; e < dim; ++e) vtmp[e] = vert[e];
1605       vtmp[d]   = (vert[d] + vertices[d] - 1) % vertices[d];
1606       supp[s++] = TupleToIndex_Private(dim, vertices, vtmp) * dim + d;
1607       supp[s++] = (vertex - numCells) * dim + d;
1608       PetscCall(DMPlexSetSupport(dm, vertex, supp));
1609     }
1610     PetscCall(DMPlexTensorPointLexicographic_Private(dim, vertices, vert));
1611   }
1612   PetscCall(DMPlexStratify(dm));
1613   /* Build coordinates */
1614   PetscCall(DMGetCoordinateSection(dm, &coordSection));
1615   PetscCall(PetscSectionSetNumFields(coordSection, 1));
1616   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1617   PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1618   for (v = numCells; v < numCells + numVertices; ++v) {
1619     PetscCall(PetscSectionSetDof(coordSection, v, dim));
1620     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1621   }
1622   PetscCall(PetscSectionSetUp(coordSection));
1623   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1624   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1625   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1626   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1627   PetscCall(VecSetBlockSize(coordinates, dim));
1628   PetscCall(VecSetType(coordinates, VECSTANDARD));
1629   PetscCall(VecGetArray(coordinates, &coords));
1630   for (d = 0; d < dim; ++d) vert[d] = 0;
1631   while (vert[dim - 1] < vertices[dim - 1]) {
1632     const PetscInt vertex = TupleToIndex_Private(dim, vertices, vert);
1633 
1634     for (d = 0; d < dim; ++d) coords[vertex * dim + d] = lower[d] + ((upper[d] - lower[d]) / vertices[d]) * vert[d];
1635     PetscCall(PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT ":", vertex));
1636     for (d = 0; d < dim; ++d) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, vert[d]));
1637     for (d = 0; d < dim; ++d) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %g", (double)PetscRealPart(coords[vertex * dim + d])));
1638     PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
1639     PetscCall(DMPlexTensorPointLexicographic_Private(dim, vertices, vert));
1640   }
1641   PetscCall(VecRestoreArray(coordinates, &coords));
1642   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1643   PetscCall(VecDestroy(&coordinates));
1644   PetscCall(PetscFree4(vertices, vert, vtmp, supp));
1645   //PetscCall(DMSetPeriodicity(dm, NULL, lower, upper));
1646   // Attach the extent
1647   {
1648     PetscContainer c;
1649     PetscInt      *extent;
1650 
1651     PetscCall(PetscMalloc1(dim, &extent));
1652     for (PetscInt d = 0; d < dim; ++d) extent[d] = edges[d];
1653     PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &c));
1654     PetscCall(PetscContainerSetUserDestroy(c, DestroyExtent_Private));
1655     PetscCall(PetscContainerSetPointer(c, extent));
1656     PetscCall(PetscObjectCompose((PetscObject)dm, "_extent", (PetscObject)c));
1657     PetscCall(PetscContainerDestroy(&c));
1658   }
1659   PetscFunctionReturn(PETSC_SUCCESS);
1660 }
1661 
1662 /*@C
1663   DMPlexCreateHypercubicMesh - Creates a periodic mesh on the tensor product of unit intervals using only vertices and edges.
1664 
1665   Collective
1666 
1667   Input Parameters:
1668 + comm  - The communicator for the DM object
1669 . dim   - The spatial dimension
1670 . edges - Number of edges per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1671 . lower - The lower left corner, or `NULL` for (0, 0, 0)
1672 - upper - The upper right corner, or `NULL` for (1, 1, 1)
1673 
1674   Output Parameter:
1675 . dm - The DM object
1676 
1677   Level: beginner
1678 
1679   Note:
1680   If you want to customize this mesh using options, you just need to
1681 .vb
1682   DMCreate(comm, &dm);
1683   DMSetType(dm, DMPLEX);
1684   DMSetFromOptions(dm);
1685 .ve
1686   and use the options on the `DMSetFromOptions()` page.
1687 
1688   The vertices are numbered is lexicographic order, and the dim edges exiting a vertex in the positive orthant are number consecutively,
1689 .vb
1690  18--0-19--2-20--4-18
1691   |     |     |     |
1692  13    15    17    13
1693   |     |     |     |
1694  24-12-25-14-26-16-24
1695   |     |     |     |
1696   7     9    11     7
1697   |     |     |     |
1698  21--6-22--8-23-10-21
1699   |     |     |     |
1700   1     3     5     1
1701   |     |     |     |
1702  18--0-19--2-20--4-18
1703 .ve
1704 
1705 .seealso: `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
1706 @*/
1707 PetscErrorCode DMPlexCreateHypercubicMesh(MPI_Comm comm, PetscInt dim, const PetscInt edges[], const PetscReal lower[], const PetscReal upper[], DM *dm)
1708 {
1709   PetscInt       *edg;
1710   PetscReal      *low, *upp;
1711   DMBoundaryType *bdt;
1712   PetscInt        d;
1713 
1714   PetscFunctionBegin;
1715   PetscCall(DMCreate(comm, dm));
1716   PetscCall(DMSetType(*dm, DMPLEX));
1717   PetscCall(PetscMalloc4(dim, &edg, dim, &low, dim, &upp, dim, &bdt));
1718   for (d = 0; d < dim; ++d) {
1719     edg[d] = edges ? edges[d] : 1;
1720     low[d] = lower ? lower[d] : 0.;
1721     upp[d] = upper ? upper[d] : 1.;
1722     bdt[d] = DM_BOUNDARY_PERIODIC;
1723   }
1724   PetscCall(DMPlexCreateHypercubicMesh_Internal(*dm, dim, low, upp, edg, bdt));
1725   PetscCall(PetscFree4(edg, low, upp, bdt));
1726   PetscFunctionReturn(PETSC_SUCCESS);
1727 }
1728 
1729 /*@
1730   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all `DM` options in the database.
1731 
1732   Logically Collective
1733 
1734   Input Parameters:
1735 + dm     - the `DM` context
1736 - prefix - the prefix to prepend to all option names
1737 
1738   Level: advanced
1739 
1740   Note:
1741   A hyphen (-) must NOT be given at the beginning of the prefix name.
1742   The first character of all runtime options is AUTOMATICALLY the hyphen.
1743 
1744 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `SNESSetFromOptions()`
1745 @*/
1746 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
1747 {
1748   DM_Plex *mesh = (DM_Plex *)dm->data;
1749 
1750   PetscFunctionBegin;
1751   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1752   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, prefix));
1753   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)mesh->partitioner, prefix));
1754   PetscFunctionReturn(PETSC_SUCCESS);
1755 }
1756 
1757 /* Remap geometry to cylinder
1758    TODO: This only works for a single refinement, then it is broken
1759 
1760      Interior square: Linear interpolation is correct
1761      The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
1762      such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
1763 
1764        phi     = arctan(y/x)
1765        d_close = sqrt(1/8 + 1/4 sin^2(phi))
1766        d_far   = sqrt(1/2 + sin^2(phi))
1767 
1768      so we remap them using
1769 
1770        x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
1771        y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
1772 
1773      If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
1774 */
1775 static void snapToCylinder(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
1776 {
1777   const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1778   const PetscReal ds2 = 0.5 * dis;
1779 
1780   if ((PetscAbsScalar(u[0]) <= ds2) && (PetscAbsScalar(u[1]) <= ds2)) {
1781     f0[0] = u[0];
1782     f0[1] = u[1];
1783   } else {
1784     PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
1785 
1786     x    = PetscRealPart(u[0]);
1787     y    = PetscRealPart(u[1]);
1788     phi  = PetscAtan2Real(y, x);
1789     sinp = PetscSinReal(phi);
1790     cosp = PetscCosReal(phi);
1791     if ((PetscAbsReal(phi) > PETSC_PI / 4.0) && (PetscAbsReal(phi) < 3.0 * PETSC_PI / 4.0)) {
1792       dc = PetscAbsReal(ds2 / sinp);
1793       df = PetscAbsReal(dis / sinp);
1794       xc = ds2 * x / PetscAbsReal(y);
1795       yc = ds2 * PetscSignReal(y);
1796     } else {
1797       dc = PetscAbsReal(ds2 / cosp);
1798       df = PetscAbsReal(dis / cosp);
1799       xc = ds2 * PetscSignReal(x);
1800       yc = ds2 * y / PetscAbsReal(x);
1801     }
1802     f0[0] = xc + (u[0] - xc) * (1.0 - dc) / (df - dc);
1803     f0[1] = yc + (u[1] - yc) * (1.0 - dc) / (df - dc);
1804   }
1805   f0[2] = u[2];
1806 }
1807 
1808 static PetscErrorCode DMPlexCreateHexCylinderMesh_Internal(DM dm, DMBoundaryType periodicZ, PetscInt Nr)
1809 {
1810   const PetscInt dim = 3;
1811   PetscInt       numCells, numVertices;
1812   PetscMPIInt    rank;
1813 
1814   PetscFunctionBegin;
1815   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
1816   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1817   PetscCall(DMSetDimension(dm, dim));
1818   /* Create topology */
1819   {
1820     PetscInt cone[8], c;
1821 
1822     numCells    = rank == 0 ? 5 : 0;
1823     numVertices = rank == 0 ? 16 : 0;
1824     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1825       numCells *= 3;
1826       numVertices = rank == 0 ? 24 : 0;
1827     }
1828     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1829     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 8));
1830     PetscCall(DMSetUp(dm));
1831     if (rank == 0) {
1832       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1833         cone[0] = 15;
1834         cone[1] = 18;
1835         cone[2] = 17;
1836         cone[3] = 16;
1837         cone[4] = 31;
1838         cone[5] = 32;
1839         cone[6] = 33;
1840         cone[7] = 34;
1841         PetscCall(DMPlexSetCone(dm, 0, cone));
1842         cone[0] = 16;
1843         cone[1] = 17;
1844         cone[2] = 24;
1845         cone[3] = 23;
1846         cone[4] = 32;
1847         cone[5] = 36;
1848         cone[6] = 37;
1849         cone[7] = 33; /* 22 25 26 21 */
1850         PetscCall(DMPlexSetCone(dm, 1, cone));
1851         cone[0] = 18;
1852         cone[1] = 27;
1853         cone[2] = 24;
1854         cone[3] = 17;
1855         cone[4] = 34;
1856         cone[5] = 33;
1857         cone[6] = 37;
1858         cone[7] = 38;
1859         PetscCall(DMPlexSetCone(dm, 2, cone));
1860         cone[0] = 29;
1861         cone[1] = 27;
1862         cone[2] = 18;
1863         cone[3] = 15;
1864         cone[4] = 35;
1865         cone[5] = 31;
1866         cone[6] = 34;
1867         cone[7] = 38;
1868         PetscCall(DMPlexSetCone(dm, 3, cone));
1869         cone[0] = 29;
1870         cone[1] = 15;
1871         cone[2] = 16;
1872         cone[3] = 23;
1873         cone[4] = 35;
1874         cone[5] = 36;
1875         cone[6] = 32;
1876         cone[7] = 31;
1877         PetscCall(DMPlexSetCone(dm, 4, cone));
1878 
1879         cone[0] = 31;
1880         cone[1] = 34;
1881         cone[2] = 33;
1882         cone[3] = 32;
1883         cone[4] = 19;
1884         cone[5] = 22;
1885         cone[6] = 21;
1886         cone[7] = 20;
1887         PetscCall(DMPlexSetCone(dm, 5, cone));
1888         cone[0] = 32;
1889         cone[1] = 33;
1890         cone[2] = 37;
1891         cone[3] = 36;
1892         cone[4] = 22;
1893         cone[5] = 25;
1894         cone[6] = 26;
1895         cone[7] = 21;
1896         PetscCall(DMPlexSetCone(dm, 6, cone));
1897         cone[0] = 34;
1898         cone[1] = 38;
1899         cone[2] = 37;
1900         cone[3] = 33;
1901         cone[4] = 20;
1902         cone[5] = 21;
1903         cone[6] = 26;
1904         cone[7] = 28;
1905         PetscCall(DMPlexSetCone(dm, 7, cone));
1906         cone[0] = 35;
1907         cone[1] = 38;
1908         cone[2] = 34;
1909         cone[3] = 31;
1910         cone[4] = 30;
1911         cone[5] = 19;
1912         cone[6] = 20;
1913         cone[7] = 28;
1914         PetscCall(DMPlexSetCone(dm, 8, cone));
1915         cone[0] = 35;
1916         cone[1] = 31;
1917         cone[2] = 32;
1918         cone[3] = 36;
1919         cone[4] = 30;
1920         cone[5] = 25;
1921         cone[6] = 22;
1922         cone[7] = 19;
1923         PetscCall(DMPlexSetCone(dm, 9, cone));
1924 
1925         cone[0] = 19;
1926         cone[1] = 20;
1927         cone[2] = 21;
1928         cone[3] = 22;
1929         cone[4] = 15;
1930         cone[5] = 16;
1931         cone[6] = 17;
1932         cone[7] = 18;
1933         PetscCall(DMPlexSetCone(dm, 10, cone));
1934         cone[0] = 22;
1935         cone[1] = 21;
1936         cone[2] = 26;
1937         cone[3] = 25;
1938         cone[4] = 16;
1939         cone[5] = 23;
1940         cone[6] = 24;
1941         cone[7] = 17;
1942         PetscCall(DMPlexSetCone(dm, 11, cone));
1943         cone[0] = 20;
1944         cone[1] = 28;
1945         cone[2] = 26;
1946         cone[3] = 21;
1947         cone[4] = 18;
1948         cone[5] = 17;
1949         cone[6] = 24;
1950         cone[7] = 27;
1951         PetscCall(DMPlexSetCone(dm, 12, cone));
1952         cone[0] = 30;
1953         cone[1] = 28;
1954         cone[2] = 20;
1955         cone[3] = 19;
1956         cone[4] = 29;
1957         cone[5] = 15;
1958         cone[6] = 18;
1959         cone[7] = 27;
1960         PetscCall(DMPlexSetCone(dm, 13, cone));
1961         cone[0] = 30;
1962         cone[1] = 19;
1963         cone[2] = 22;
1964         cone[3] = 25;
1965         cone[4] = 29;
1966         cone[5] = 23;
1967         cone[6] = 16;
1968         cone[7] = 15;
1969         PetscCall(DMPlexSetCone(dm, 14, cone));
1970       } else {
1971         cone[0] = 5;
1972         cone[1] = 8;
1973         cone[2] = 7;
1974         cone[3] = 6;
1975         cone[4] = 9;
1976         cone[5] = 12;
1977         cone[6] = 11;
1978         cone[7] = 10;
1979         PetscCall(DMPlexSetCone(dm, 0, cone));
1980         cone[0] = 6;
1981         cone[1] = 7;
1982         cone[2] = 14;
1983         cone[3] = 13;
1984         cone[4] = 12;
1985         cone[5] = 15;
1986         cone[6] = 16;
1987         cone[7] = 11;
1988         PetscCall(DMPlexSetCone(dm, 1, cone));
1989         cone[0] = 8;
1990         cone[1] = 17;
1991         cone[2] = 14;
1992         cone[3] = 7;
1993         cone[4] = 10;
1994         cone[5] = 11;
1995         cone[6] = 16;
1996         cone[7] = 18;
1997         PetscCall(DMPlexSetCone(dm, 2, cone));
1998         cone[0] = 19;
1999         cone[1] = 17;
2000         cone[2] = 8;
2001         cone[3] = 5;
2002         cone[4] = 20;
2003         cone[5] = 9;
2004         cone[6] = 10;
2005         cone[7] = 18;
2006         PetscCall(DMPlexSetCone(dm, 3, cone));
2007         cone[0] = 19;
2008         cone[1] = 5;
2009         cone[2] = 6;
2010         cone[3] = 13;
2011         cone[4] = 20;
2012         cone[5] = 15;
2013         cone[6] = 12;
2014         cone[7] = 9;
2015         PetscCall(DMPlexSetCone(dm, 4, cone));
2016       }
2017     }
2018     PetscCall(DMPlexSymmetrize(dm));
2019     PetscCall(DMPlexStratify(dm));
2020   }
2021   /* Create cube geometry */
2022   {
2023     Vec             coordinates;
2024     PetscSection    coordSection;
2025     PetscScalar    *coords;
2026     PetscInt        coordSize, v;
2027     const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
2028     const PetscReal ds2 = dis / 2.0;
2029 
2030     /* Build coordinates */
2031     PetscCall(DMGetCoordinateSection(dm, &coordSection));
2032     PetscCall(PetscSectionSetNumFields(coordSection, 1));
2033     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
2034     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
2035     for (v = numCells; v < numCells + numVertices; ++v) {
2036       PetscCall(PetscSectionSetDof(coordSection, v, dim));
2037       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
2038     }
2039     PetscCall(PetscSectionSetUp(coordSection));
2040     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2041     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2042     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2043     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2044     PetscCall(VecSetBlockSize(coordinates, dim));
2045     PetscCall(VecSetType(coordinates, VECSTANDARD));
2046     PetscCall(VecGetArray(coordinates, &coords));
2047     if (rank == 0) {
2048       coords[0 * dim + 0]  = -ds2;
2049       coords[0 * dim + 1]  = -ds2;
2050       coords[0 * dim + 2]  = 0.0;
2051       coords[1 * dim + 0]  = ds2;
2052       coords[1 * dim + 1]  = -ds2;
2053       coords[1 * dim + 2]  = 0.0;
2054       coords[2 * dim + 0]  = ds2;
2055       coords[2 * dim + 1]  = ds2;
2056       coords[2 * dim + 2]  = 0.0;
2057       coords[3 * dim + 0]  = -ds2;
2058       coords[3 * dim + 1]  = ds2;
2059       coords[3 * dim + 2]  = 0.0;
2060       coords[4 * dim + 0]  = -ds2;
2061       coords[4 * dim + 1]  = -ds2;
2062       coords[4 * dim + 2]  = 1.0;
2063       coords[5 * dim + 0]  = -ds2;
2064       coords[5 * dim + 1]  = ds2;
2065       coords[5 * dim + 2]  = 1.0;
2066       coords[6 * dim + 0]  = ds2;
2067       coords[6 * dim + 1]  = ds2;
2068       coords[6 * dim + 2]  = 1.0;
2069       coords[7 * dim + 0]  = ds2;
2070       coords[7 * dim + 1]  = -ds2;
2071       coords[7 * dim + 2]  = 1.0;
2072       coords[8 * dim + 0]  = dis;
2073       coords[8 * dim + 1]  = -dis;
2074       coords[8 * dim + 2]  = 0.0;
2075       coords[9 * dim + 0]  = dis;
2076       coords[9 * dim + 1]  = dis;
2077       coords[9 * dim + 2]  = 0.0;
2078       coords[10 * dim + 0] = dis;
2079       coords[10 * dim + 1] = -dis;
2080       coords[10 * dim + 2] = 1.0;
2081       coords[11 * dim + 0] = dis;
2082       coords[11 * dim + 1] = dis;
2083       coords[11 * dim + 2] = 1.0;
2084       coords[12 * dim + 0] = -dis;
2085       coords[12 * dim + 1] = dis;
2086       coords[12 * dim + 2] = 0.0;
2087       coords[13 * dim + 0] = -dis;
2088       coords[13 * dim + 1] = dis;
2089       coords[13 * dim + 2] = 1.0;
2090       coords[14 * dim + 0] = -dis;
2091       coords[14 * dim + 1] = -dis;
2092       coords[14 * dim + 2] = 0.0;
2093       coords[15 * dim + 0] = -dis;
2094       coords[15 * dim + 1] = -dis;
2095       coords[15 * dim + 2] = 1.0;
2096       if (periodicZ == DM_BOUNDARY_PERIODIC) {
2097         /* 15 31 19 */ coords[16 * dim + 0] = -ds2;
2098         coords[16 * dim + 1]                = -ds2;
2099         coords[16 * dim + 2]                = 0.5;
2100         /* 16 32 22 */ coords[17 * dim + 0] = ds2;
2101         coords[17 * dim + 1]                = -ds2;
2102         coords[17 * dim + 2]                = 0.5;
2103         /* 17 33 21 */ coords[18 * dim + 0] = ds2;
2104         coords[18 * dim + 1]                = ds2;
2105         coords[18 * dim + 2]                = 0.5;
2106         /* 18 34 20 */ coords[19 * dim + 0] = -ds2;
2107         coords[19 * dim + 1]                = ds2;
2108         coords[19 * dim + 2]                = 0.5;
2109         /* 29 35 30 */ coords[20 * dim + 0] = -dis;
2110         coords[20 * dim + 1]                = -dis;
2111         coords[20 * dim + 2]                = 0.5;
2112         /* 23 36 25 */ coords[21 * dim + 0] = dis;
2113         coords[21 * dim + 1]                = -dis;
2114         coords[21 * dim + 2]                = 0.5;
2115         /* 24 37 26 */ coords[22 * dim + 0] = dis;
2116         coords[22 * dim + 1]                = dis;
2117         coords[22 * dim + 2]                = 0.5;
2118         /* 27 38 28 */ coords[23 * dim + 0] = -dis;
2119         coords[23 * dim + 1]                = dis;
2120         coords[23 * dim + 2]                = 0.5;
2121       }
2122     }
2123     PetscCall(VecRestoreArray(coordinates, &coords));
2124     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2125     PetscCall(VecDestroy(&coordinates));
2126   }
2127   /* Create periodicity */
2128   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
2129     PetscReal L[3]       = {-1., -1., 0.};
2130     PetscReal maxCell[3] = {-1., -1., 0.};
2131     PetscReal lower[3]   = {0.0, 0.0, 0.0};
2132     PetscReal upper[3]   = {1.0, 1.0, 1.5};
2133     PetscInt  numZCells  = 3;
2134 
2135     L[2]       = upper[2] - lower[2];
2136     maxCell[2] = 1.1 * (L[2] / numZCells);
2137     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
2138   }
2139   {
2140     DM          cdm;
2141     PetscDS     cds;
2142     PetscScalar c[2] = {1.0, 1.0};
2143 
2144     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_TRUE, NULL));
2145     PetscCall(DMGetCoordinateDM(dm, &cdm));
2146     PetscCall(DMGetDS(cdm, &cds));
2147     PetscCall(PetscDSSetConstants(cds, 2, c));
2148   }
2149   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
2150 
2151   /* Wait for coordinate creation before doing in-place modification */
2152   PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2153 
2154   char        oldprefix[PETSC_MAX_PATH_LEN];
2155   const char *prefix;
2156 
2157   PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
2158   PetscCall(PetscStrncpy(oldprefix, prefix, PETSC_MAX_PATH_LEN));
2159   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, "petsc_cyl_ref_"));
2160   for (PetscInt r = 0; r < PetscMax(0, Nr); ++r) {
2161     DM rdm;
2162 
2163     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
2164     PetscCall(DMPlexReplace_Internal(dm, &rdm));
2165   }
2166   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, oldprefix));
2167   PetscCall(DMPlexRemapGeometry(dm, 0.0, snapToCylinder));
2168 
2169   DMLabel         bdlabel, edgelabel;
2170   IS              faceIS;
2171   const PetscInt *faces;
2172   PetscInt        Nf;
2173 
2174   PetscCall(DMCreateLabel(dm, "marker"));
2175   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
2176   PetscCall(DMCreateLabel(dm, "generatrix"));
2177   PetscCall(DMGetLabel(dm, "generatrix", &edgelabel));
2178   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
2179   // Remove faces on top and bottom
2180   PetscCall(DMLabelGetStratumIS(bdlabel, 1, &faceIS));
2181   PetscCall(ISGetLocalSize(faceIS, &Nf));
2182   PetscCall(ISGetIndices(faceIS, &faces));
2183   for (PetscInt f = 0; f < Nf; ++f) {
2184     PetscReal vol, normal[3];
2185 
2186     PetscCall(DMPlexComputeCellGeometryFVM(dm, faces[f], &vol, NULL, normal));
2187     if (PetscAbsReal(normal[2]) < PETSC_SMALL) PetscCall(DMLabelSetValue(edgelabel, faces[f], 1));
2188   }
2189   PetscCall(ISRestoreIndices(faceIS, &faces));
2190   PetscCall(ISDestroy(&faceIS));
2191   PetscCall(DMPlexLabelComplete(dm, bdlabel));
2192   PetscCall(DMPlexLabelComplete(dm, edgelabel));
2193   PetscFunctionReturn(PETSC_SUCCESS);
2194 }
2195 
2196 /*@
2197   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
2198 
2199   Collective
2200 
2201   Input Parameters:
2202 + comm      - The communicator for the `DM` object
2203 . periodicZ - The boundary type for the Z direction
2204 - Nr        - The number of refinements to carry out
2205 
2206   Output Parameter:
2207 . dm - The `DM` object
2208 
2209   Level: beginner
2210 
2211   Note:
2212   Here is the output numbering looking from the bottom of the cylinder\:
2213 .vb
2214        17-----14
2215         |     |
2216         |  2  |
2217         |     |
2218  17-----8-----7-----14
2219   |     |     |     |
2220   |  3  |  0  |  1  |
2221   |     |     |     |
2222  19-----5-----6-----13
2223         |     |
2224         |  4  |
2225         |     |
2226        19-----13
2227 
2228  and up through the top
2229 
2230        18-----16
2231         |     |
2232         |  2  |
2233         |     |
2234  18----10----11-----16
2235   |     |     |     |
2236   |  3  |  0  |  1  |
2237   |     |     |     |
2238  20-----9----12-----15
2239         |     |
2240         |  4  |
2241         |     |
2242        20-----15
2243 .ve
2244 
2245 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2246 @*/
2247 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, DMBoundaryType periodicZ, PetscInt Nr, DM *dm)
2248 {
2249   PetscFunctionBegin;
2250   PetscAssertPointer(dm, 4);
2251   PetscCall(DMCreate(comm, dm));
2252   PetscCall(DMSetType(*dm, DMPLEX));
2253   PetscCall(DMPlexCreateHexCylinderMesh_Internal(*dm, periodicZ, Nr));
2254   PetscFunctionReturn(PETSC_SUCCESS);
2255 }
2256 
2257 static PetscErrorCode DMPlexCreateWedgeCylinderMesh_Internal(DM dm, PetscInt n, PetscBool interpolate)
2258 {
2259   const PetscInt dim = 3;
2260   PetscInt       numCells, numVertices, v;
2261   PetscMPIInt    rank;
2262 
2263   PetscFunctionBegin;
2264   PetscCheck(n >= 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %" PetscInt_FMT " cannot be negative", n);
2265   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
2266   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2267   PetscCall(DMSetDimension(dm, dim));
2268   /* Must create the celltype label here so that we do not automatically try to compute the types */
2269   PetscCall(DMCreateLabel(dm, "celltype"));
2270   /* Create topology */
2271   {
2272     PetscInt cone[6], c;
2273 
2274     numCells    = rank == 0 ? n : 0;
2275     numVertices = rank == 0 ? 2 * (n + 1) : 0;
2276     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
2277     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
2278     PetscCall(DMSetUp(dm));
2279     for (c = 0; c < numCells; c++) {
2280       cone[0] = c + n * 1;
2281       cone[1] = (c + 1) % n + n * 1;
2282       cone[2] = 0 + 3 * n;
2283       cone[3] = c + n * 2;
2284       cone[4] = (c + 1) % n + n * 2;
2285       cone[5] = 1 + 3 * n;
2286       PetscCall(DMPlexSetCone(dm, c, cone));
2287       PetscCall(DMPlexSetCellType(dm, c, DM_POLYTOPE_TRI_PRISM_TENSOR));
2288     }
2289     PetscCall(DMPlexSymmetrize(dm));
2290     PetscCall(DMPlexStratify(dm));
2291   }
2292   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetCellType(dm, v, DM_POLYTOPE_POINT));
2293   /* Create cylinder geometry */
2294   {
2295     Vec          coordinates;
2296     PetscSection coordSection;
2297     PetscScalar *coords;
2298     PetscInt     coordSize, c;
2299 
2300     /* Build coordinates */
2301     PetscCall(DMGetCoordinateSection(dm, &coordSection));
2302     PetscCall(PetscSectionSetNumFields(coordSection, 1));
2303     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
2304     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
2305     for (v = numCells; v < numCells + numVertices; ++v) {
2306       PetscCall(PetscSectionSetDof(coordSection, v, dim));
2307       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
2308     }
2309     PetscCall(PetscSectionSetUp(coordSection));
2310     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2311     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2312     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2313     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2314     PetscCall(VecSetBlockSize(coordinates, dim));
2315     PetscCall(VecSetType(coordinates, VECSTANDARD));
2316     PetscCall(VecGetArray(coordinates, &coords));
2317     for (c = 0; c < numCells; c++) {
2318       coords[(c + 0 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
2319       coords[(c + 0 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
2320       coords[(c + 0 * n) * dim + 2] = 1.0;
2321       coords[(c + 1 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
2322       coords[(c + 1 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
2323       coords[(c + 1 * n) * dim + 2] = 0.0;
2324     }
2325     if (rank == 0) {
2326       coords[(2 * n + 0) * dim + 0] = 0.0;
2327       coords[(2 * n + 0) * dim + 1] = 0.0;
2328       coords[(2 * n + 0) * dim + 2] = 1.0;
2329       coords[(2 * n + 1) * dim + 0] = 0.0;
2330       coords[(2 * n + 1) * dim + 1] = 0.0;
2331       coords[(2 * n + 1) * dim + 2] = 0.0;
2332     }
2333     PetscCall(VecRestoreArray(coordinates, &coords));
2334     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2335     PetscCall(VecDestroy(&coordinates));
2336   }
2337   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
2338   /* Interpolate */
2339   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2340   PetscFunctionReturn(PETSC_SUCCESS);
2341 }
2342 
2343 /*@
2344   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
2345 
2346   Collective
2347 
2348   Input Parameters:
2349 + comm        - The communicator for the `DM` object
2350 . n           - The number of wedges around the origin
2351 - interpolate - Create edges and faces
2352 
2353   Output Parameter:
2354 . dm - The `DM` object
2355 
2356   Level: beginner
2357 
2358 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2359 @*/
2360 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
2361 {
2362   PetscFunctionBegin;
2363   PetscAssertPointer(dm, 4);
2364   PetscCall(DMCreate(comm, dm));
2365   PetscCall(DMSetType(*dm, DMPLEX));
2366   PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(*dm, n, interpolate));
2367   PetscFunctionReturn(PETSC_SUCCESS);
2368 }
2369 
2370 static inline PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2371 {
2372   PetscReal prod = 0.0;
2373   PetscInt  i;
2374   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
2375   return PetscSqrtReal(prod);
2376 }
2377 
2378 static inline PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2379 {
2380   PetscReal prod = 0.0;
2381   PetscInt  i;
2382   for (i = 0; i < dim; ++i) prod += x[i] * y[i];
2383   return prod;
2384 }
2385 
2386 /* The first constant is the sphere radius */
2387 static void snapToSphere(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
2388 {
2389   PetscReal r     = PetscRealPart(constants[0]);
2390   PetscReal norm2 = 0.0, fac;
2391   PetscInt  n     = uOff[1] - uOff[0], d;
2392 
2393   for (d = 0; d < n; ++d) norm2 += PetscSqr(PetscRealPart(u[d]));
2394   fac = r / PetscSqrtReal(norm2);
2395   for (d = 0; d < n; ++d) f0[d] = u[d] * fac;
2396 }
2397 
2398 static PetscErrorCode DMPlexCreateSphereMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, PetscReal R)
2399 {
2400   const PetscInt embedDim = dim + 1;
2401   PetscSection   coordSection;
2402   Vec            coordinates;
2403   PetscScalar   *coords;
2404   PetscReal     *coordsIn;
2405   PetscInt       numCells, numEdges, numVerts = 0, firstVertex = 0, v, firstEdge, coordSize, d, e;
2406   PetscMPIInt    rank;
2407 
2408   PetscFunctionBegin;
2409   PetscValidLogicalCollectiveBool(dm, simplex, 3);
2410   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
2411   PetscCall(DMSetDimension(dm, dim));
2412   PetscCall(DMSetCoordinateDim(dm, dim + 1));
2413   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2414   switch (dim) {
2415   case 1:
2416     numCells = 16;
2417     numVerts = numCells;
2418 
2419     // Build Topology
2420     PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2421     for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2422     PetscCall(DMSetUp(dm));
2423     for (PetscInt c = 0; c < numCells; ++c) {
2424       PetscInt cone[2];
2425 
2426       cone[0] = c + numCells;
2427       cone[1] = (c + 1) % numVerts + numCells;
2428       PetscCall(DMPlexSetCone(dm, c, cone));
2429     }
2430     PetscCall(DMPlexSymmetrize(dm));
2431     PetscCall(DMPlexStratify(dm));
2432     PetscCall(PetscMalloc1(numVerts * embedDim, &coordsIn));
2433     for (PetscInt v = 0; v < numVerts; ++v) {
2434       const PetscReal rad = 2. * PETSC_PI * v / numVerts;
2435 
2436       coordsIn[v * embedDim + 0] = PetscCosReal(rad);
2437       coordsIn[v * embedDim + 1] = PetscSinReal(rad);
2438     }
2439     break;
2440   case 2:
2441     if (simplex) {
2442       const PetscReal radius    = PetscSqrtReal(1 + PETSC_PHI * PETSC_PHI) / (1.0 + PETSC_PHI);
2443       const PetscReal edgeLen   = 2.0 / (1.0 + PETSC_PHI) * (R / radius);
2444       const PetscInt  degree    = 5;
2445       PetscReal       vertex[3] = {0.0, 1.0 / (1.0 + PETSC_PHI), PETSC_PHI / (1.0 + PETSC_PHI)};
2446       PetscInt        s[3]      = {1, 1, 1};
2447       PetscInt        cone[3];
2448       PetscInt       *graph;
2449 
2450       vertex[0] *= R / radius;
2451       vertex[1] *= R / radius;
2452       vertex[2] *= R / radius;
2453       numCells    = rank == 0 ? 20 : 0;
2454       numVerts    = rank == 0 ? 12 : 0;
2455       firstVertex = numCells;
2456       /* Use icosahedron, which for a R-sphere has coordinates which are all cyclic permutations of
2457 
2458            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
2459 
2460          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2461          length is then given by 2/(1+\phi) = 2 * 0.38197 = 0.76393.
2462       */
2463       /* Construct vertices */
2464       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2465       if (rank == 0) {
2466         for (PetscInt p = 0, i = 0; p < embedDim; ++p) {
2467           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2468             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2469               for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertex[(d + p) % embedDim];
2470               ++i;
2471             }
2472           }
2473         }
2474       }
2475       /* Construct graph */
2476       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2477       for (PetscInt i = 0; i < numVerts; ++i) {
2478         PetscInt k = 0;
2479         for (PetscInt j = 0; j < numVerts; ++j) {
2480           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2481             graph[i * numVerts + j] = 1;
2482             ++k;
2483           }
2484         }
2485         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid icosahedron, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2486       }
2487       /* Build Topology */
2488       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2489       for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2490       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2491       /* Cells */
2492       for (PetscInt i = 0, c = 0; i < numVerts; ++i) {
2493         for (PetscInt j = 0; j < i; ++j) {
2494           for (PetscInt k = 0; k < j; ++k) {
2495             if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i]) {
2496               cone[0] = firstVertex + i;
2497               cone[1] = firstVertex + j;
2498               cone[2] = firstVertex + k;
2499               /* Check orientation */
2500               {
2501                 const PetscInt epsilon[3][3][3] = {
2502                   {{0, 0, 0},  {0, 0, 1},  {0, -1, 0}},
2503                   {{0, 0, -1}, {0, 0, 0},  {1, 0, 0} },
2504                   {{0, 1, 0},  {-1, 0, 0}, {0, 0, 0} }
2505                 };
2506                 PetscReal normal[3];
2507                 PetscInt  e, f;
2508 
2509                 for (d = 0; d < embedDim; ++d) {
2510                   normal[d] = 0.0;
2511                   for (e = 0; e < embedDim; ++e) {
2512                     for (f = 0; f < embedDim; ++f) normal[d] += epsilon[d][e][f] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]);
2513                   }
2514                 }
2515                 if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2516                   PetscInt tmp = cone[1];
2517                   cone[1]      = cone[2];
2518                   cone[2]      = tmp;
2519                 }
2520               }
2521               PetscCall(DMPlexSetCone(dm, c++, cone));
2522             }
2523           }
2524         }
2525       }
2526       PetscCall(DMPlexSymmetrize(dm));
2527       PetscCall(DMPlexStratify(dm));
2528       PetscCall(PetscFree(graph));
2529     } else {
2530       /*
2531         12-21--13
2532          |     |
2533         25  4  24
2534          |     |
2535   12-25--9-16--8-24--13
2536    |     |     |     |
2537   23  5 17  0 15  3  22
2538    |     |     |     |
2539   10-20--6-14--7-19--11
2540          |     |
2541         20  1  19
2542          |     |
2543         10-18--11
2544          |     |
2545         23  2  22
2546          |     |
2547         12-21--13
2548        */
2549       PetscInt cone[4], ornt[4];
2550 
2551       numCells    = rank == 0 ? 6 : 0;
2552       numEdges    = rank == 0 ? 12 : 0;
2553       numVerts    = rank == 0 ? 8 : 0;
2554       firstVertex = numCells;
2555       firstEdge   = numCells + numVerts;
2556       /* Build Topology */
2557       PetscCall(DMPlexSetChart(dm, 0, numCells + numEdges + numVerts));
2558       for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 4));
2559       for (e = firstEdge; e < firstEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
2560       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2561       if (rank == 0) {
2562         /* Cell 0 */
2563         cone[0] = 14;
2564         cone[1] = 15;
2565         cone[2] = 16;
2566         cone[3] = 17;
2567         PetscCall(DMPlexSetCone(dm, 0, cone));
2568         ornt[0] = 0;
2569         ornt[1] = 0;
2570         ornt[2] = 0;
2571         ornt[3] = 0;
2572         PetscCall(DMPlexSetConeOrientation(dm, 0, ornt));
2573         /* Cell 1 */
2574         cone[0] = 18;
2575         cone[1] = 19;
2576         cone[2] = 14;
2577         cone[3] = 20;
2578         PetscCall(DMPlexSetCone(dm, 1, cone));
2579         ornt[0] = 0;
2580         ornt[1] = 0;
2581         ornt[2] = -1;
2582         ornt[3] = 0;
2583         PetscCall(DMPlexSetConeOrientation(dm, 1, ornt));
2584         /* Cell 2 */
2585         cone[0] = 21;
2586         cone[1] = 22;
2587         cone[2] = 18;
2588         cone[3] = 23;
2589         PetscCall(DMPlexSetCone(dm, 2, cone));
2590         ornt[0] = 0;
2591         ornt[1] = 0;
2592         ornt[2] = -1;
2593         ornt[3] = 0;
2594         PetscCall(DMPlexSetConeOrientation(dm, 2, ornt));
2595         /* Cell 3 */
2596         cone[0] = 19;
2597         cone[1] = 22;
2598         cone[2] = 24;
2599         cone[3] = 15;
2600         PetscCall(DMPlexSetCone(dm, 3, cone));
2601         ornt[0] = -1;
2602         ornt[1] = -1;
2603         ornt[2] = 0;
2604         ornt[3] = -1;
2605         PetscCall(DMPlexSetConeOrientation(dm, 3, ornt));
2606         /* Cell 4 */
2607         cone[0] = 16;
2608         cone[1] = 24;
2609         cone[2] = 21;
2610         cone[3] = 25;
2611         PetscCall(DMPlexSetCone(dm, 4, cone));
2612         ornt[0] = -1;
2613         ornt[1] = -1;
2614         ornt[2] = -1;
2615         ornt[3] = 0;
2616         PetscCall(DMPlexSetConeOrientation(dm, 4, ornt));
2617         /* Cell 5 */
2618         cone[0] = 20;
2619         cone[1] = 17;
2620         cone[2] = 25;
2621         cone[3] = 23;
2622         PetscCall(DMPlexSetCone(dm, 5, cone));
2623         ornt[0] = -1;
2624         ornt[1] = -1;
2625         ornt[2] = -1;
2626         ornt[3] = -1;
2627         PetscCall(DMPlexSetConeOrientation(dm, 5, ornt));
2628         /* Edges */
2629         cone[0] = 6;
2630         cone[1] = 7;
2631         PetscCall(DMPlexSetCone(dm, 14, cone));
2632         cone[0] = 7;
2633         cone[1] = 8;
2634         PetscCall(DMPlexSetCone(dm, 15, cone));
2635         cone[0] = 8;
2636         cone[1] = 9;
2637         PetscCall(DMPlexSetCone(dm, 16, cone));
2638         cone[0] = 9;
2639         cone[1] = 6;
2640         PetscCall(DMPlexSetCone(dm, 17, cone));
2641         cone[0] = 10;
2642         cone[1] = 11;
2643         PetscCall(DMPlexSetCone(dm, 18, cone));
2644         cone[0] = 11;
2645         cone[1] = 7;
2646         PetscCall(DMPlexSetCone(dm, 19, cone));
2647         cone[0] = 6;
2648         cone[1] = 10;
2649         PetscCall(DMPlexSetCone(dm, 20, cone));
2650         cone[0] = 12;
2651         cone[1] = 13;
2652         PetscCall(DMPlexSetCone(dm, 21, cone));
2653         cone[0] = 13;
2654         cone[1] = 11;
2655         PetscCall(DMPlexSetCone(dm, 22, cone));
2656         cone[0] = 10;
2657         cone[1] = 12;
2658         PetscCall(DMPlexSetCone(dm, 23, cone));
2659         cone[0] = 13;
2660         cone[1] = 8;
2661         PetscCall(DMPlexSetCone(dm, 24, cone));
2662         cone[0] = 12;
2663         cone[1] = 9;
2664         PetscCall(DMPlexSetCone(dm, 25, cone));
2665       }
2666       PetscCall(DMPlexSymmetrize(dm));
2667       PetscCall(DMPlexStratify(dm));
2668       /* Build coordinates */
2669       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2670       if (rank == 0) {
2671         coordsIn[0 * embedDim + 0] = -R;
2672         coordsIn[0 * embedDim + 1] = R;
2673         coordsIn[0 * embedDim + 2] = -R;
2674         coordsIn[1 * embedDim + 0] = R;
2675         coordsIn[1 * embedDim + 1] = R;
2676         coordsIn[1 * embedDim + 2] = -R;
2677         coordsIn[2 * embedDim + 0] = R;
2678         coordsIn[2 * embedDim + 1] = -R;
2679         coordsIn[2 * embedDim + 2] = -R;
2680         coordsIn[3 * embedDim + 0] = -R;
2681         coordsIn[3 * embedDim + 1] = -R;
2682         coordsIn[3 * embedDim + 2] = -R;
2683         coordsIn[4 * embedDim + 0] = -R;
2684         coordsIn[4 * embedDim + 1] = R;
2685         coordsIn[4 * embedDim + 2] = R;
2686         coordsIn[5 * embedDim + 0] = R;
2687         coordsIn[5 * embedDim + 1] = R;
2688         coordsIn[5 * embedDim + 2] = R;
2689         coordsIn[6 * embedDim + 0] = -R;
2690         coordsIn[6 * embedDim + 1] = -R;
2691         coordsIn[6 * embedDim + 2] = R;
2692         coordsIn[7 * embedDim + 0] = R;
2693         coordsIn[7 * embedDim + 1] = -R;
2694         coordsIn[7 * embedDim + 2] = R;
2695       }
2696     }
2697     break;
2698   case 3:
2699     if (simplex) {
2700       const PetscReal edgeLen         = 1.0 / PETSC_PHI;
2701       PetscReal       vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
2702       PetscReal       vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
2703       PetscReal       vertexC[4]      = {0.5, 0.5 * PETSC_PHI, 0.5 / PETSC_PHI, 0.0};
2704       const PetscInt  degree          = 12;
2705       PetscInt        s[4]            = {1, 1, 1};
2706       PetscInt        evenPerm[12][4] = {
2707         {0, 1, 2, 3},
2708         {0, 2, 3, 1},
2709         {0, 3, 1, 2},
2710         {1, 0, 3, 2},
2711         {1, 2, 0, 3},
2712         {1, 3, 2, 0},
2713         {2, 0, 1, 3},
2714         {2, 1, 3, 0},
2715         {2, 3, 0, 1},
2716         {3, 0, 2, 1},
2717         {3, 1, 0, 2},
2718         {3, 2, 1, 0}
2719       };
2720       PetscInt  cone[4];
2721       PetscInt *graph, p, i, j, k, l;
2722 
2723       vertexA[0] *= R;
2724       vertexA[1] *= R;
2725       vertexA[2] *= R;
2726       vertexA[3] *= R;
2727       vertexB[0] *= R;
2728       vertexB[1] *= R;
2729       vertexB[2] *= R;
2730       vertexB[3] *= R;
2731       vertexC[0] *= R;
2732       vertexC[1] *= R;
2733       vertexC[2] *= R;
2734       vertexC[3] *= R;
2735       numCells    = rank == 0 ? 600 : 0;
2736       numVerts    = rank == 0 ? 120 : 0;
2737       firstVertex = numCells;
2738       /* Use the 600-cell, which for a unit sphere has coordinates which are
2739 
2740            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
2741                (\pm 1,    0,       0,      0)  all cyclic permutations   8
2742            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
2743 
2744          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2745          length is then given by 1/\phi = 0.61803.
2746 
2747          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
2748          http://mathworld.wolfram.com/600-Cell.html
2749       */
2750       /* Construct vertices */
2751       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2752       i = 0;
2753       if (rank == 0) {
2754         for (s[0] = -1; s[0] < 2; s[0] += 2) {
2755           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2756             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2757               for (s[3] = -1; s[3] < 2; s[3] += 2) {
2758                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[d] * vertexA[d];
2759                 ++i;
2760               }
2761             }
2762           }
2763         }
2764         for (p = 0; p < embedDim; ++p) {
2765           s[1] = s[2] = s[3] = 1;
2766           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2767             for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertexB[(d + p) % embedDim];
2768             ++i;
2769           }
2770         }
2771         for (p = 0; p < 12; ++p) {
2772           s[3] = 1;
2773           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2774             for (s[1] = -1; s[1] < 2; s[1] += 2) {
2775               for (s[2] = -1; s[2] < 2; s[2] += 2) {
2776                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[evenPerm[p][d]] * vertexC[evenPerm[p][d]];
2777                 ++i;
2778               }
2779             }
2780           }
2781         }
2782       }
2783       PetscCheck(i == numVerts, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertices %" PetscInt_FMT " != %" PetscInt_FMT, i, numVerts);
2784       /* Construct graph */
2785       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2786       for (i = 0; i < numVerts; ++i) {
2787         for (j = 0, k = 0; j < numVerts; ++j) {
2788           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2789             graph[i * numVerts + j] = 1;
2790             ++k;
2791           }
2792         }
2793         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2794       }
2795       /* Build Topology */
2796       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2797       for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2798       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2799       /* Cells */
2800       if (rank == 0) {
2801         for (PetscInt i = 0, c = 0; i < numVerts; ++i) {
2802           for (j = 0; j < i; ++j) {
2803             for (k = 0; k < j; ++k) {
2804               for (l = 0; l < k; ++l) {
2805                 if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i] && graph[l * numVerts + i] && graph[l * numVerts + j] && graph[l * numVerts + k]) {
2806                   cone[0] = firstVertex + i;
2807                   cone[1] = firstVertex + j;
2808                   cone[2] = firstVertex + k;
2809                   cone[3] = firstVertex + l;
2810                   /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
2811                   {
2812                     const PetscInt epsilon[4][4][4][4] = {
2813                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, -1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 0, 0}, {0, 1, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 1, 0}, {0, -1, 0, 0}, {0, 0, 0, 0}}},
2814 
2815                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}}, {{0, 0, -1, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}}},
2816 
2817                       {{{0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, 0, 0}, {0, -1, 0, 0}}, {{0, 0, 0, -1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 1, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}},
2818 
2819                       {{{0, 0, 0, 0}, {0, 0, -1, 0}, {0, 1, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 1, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}}, {{0, -1, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}} }
2820                     };
2821                     PetscReal normal[4];
2822                     PetscInt  e, f, g;
2823 
2824                     for (d = 0; d < embedDim; ++d) {
2825                       normal[d] = 0.0;
2826                       for (e = 0; e < embedDim; ++e) {
2827                         for (f = 0; f < embedDim; ++f) {
2828                           for (g = 0; g < embedDim; ++g) {
2829                             normal[d] += epsilon[d][e][f][g] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]) * (coordsIn[l * embedDim + f] - coordsIn[i * embedDim + f]);
2830                           }
2831                         }
2832                       }
2833                     }
2834                     if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2835                       PetscInt tmp = cone[1];
2836                       cone[1]      = cone[2];
2837                       cone[2]      = tmp;
2838                     }
2839                   }
2840                   PetscCall(DMPlexSetCone(dm, c++, cone));
2841                 }
2842               }
2843             }
2844           }
2845         }
2846       }
2847       PetscCall(DMPlexSymmetrize(dm));
2848       PetscCall(DMPlexStratify(dm));
2849       PetscCall(PetscFree(graph));
2850     }
2851     break;
2852   default:
2853     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unsupported dimension for sphere: %" PetscInt_FMT, dim);
2854   }
2855   /* Create coordinates */
2856   PetscCall(DMGetCoordinateSection(dm, &coordSection));
2857   PetscCall(PetscSectionSetNumFields(coordSection, 1));
2858   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, embedDim));
2859   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVerts));
2860   for (v = firstVertex; v < firstVertex + numVerts; ++v) {
2861     PetscCall(PetscSectionSetDof(coordSection, v, embedDim));
2862     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, embedDim));
2863   }
2864   PetscCall(PetscSectionSetUp(coordSection));
2865   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2866   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2867   PetscCall(VecSetBlockSize(coordinates, embedDim));
2868   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2869   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2870   PetscCall(VecSetType(coordinates, VECSTANDARD));
2871   PetscCall(VecGetArray(coordinates, &coords));
2872   for (v = 0; v < numVerts; ++v)
2873     for (d = 0; d < embedDim; ++d) coords[v * embedDim + d] = coordsIn[v * embedDim + d];
2874   PetscCall(VecRestoreArray(coordinates, &coords));
2875   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2876   PetscCall(VecDestroy(&coordinates));
2877   PetscCall(PetscFree(coordsIn));
2878   {
2879     DM          cdm;
2880     PetscDS     cds;
2881     PetscScalar c = R;
2882 
2883     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_TRUE, snapToSphere));
2884     PetscCall(DMGetCoordinateDM(dm, &cdm));
2885     PetscCall(DMGetDS(cdm, &cds));
2886     PetscCall(PetscDSSetConstants(cds, 1, &c));
2887   }
2888   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
2889   /* Wait for coordinate creation before doing in-place modification */
2890   if (simplex) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2891   PetscFunctionReturn(PETSC_SUCCESS);
2892 }
2893 
2894 typedef void (*TPSEvaluateFunc)(const PetscReal[], PetscReal *, PetscReal[], PetscReal (*)[3]);
2895 
2896 /*
2897  The Schwarz P implicit surface is
2898 
2899      f(x) = cos(x0) + cos(x1) + cos(x2) = 0
2900 */
2901 static void TPSEvaluate_SchwarzP(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2902 {
2903   PetscReal c[3] = {PetscCosReal(y[0] * PETSC_PI), PetscCosReal(y[1] * PETSC_PI), PetscCosReal(y[2] * PETSC_PI)};
2904   PetscReal g[3] = {-PetscSinReal(y[0] * PETSC_PI), -PetscSinReal(y[1] * PETSC_PI), -PetscSinReal(y[2] * PETSC_PI)};
2905   f[0]           = c[0] + c[1] + c[2];
2906   for (PetscInt i = 0; i < 3; i++) {
2907     grad[i] = PETSC_PI * g[i];
2908     for (PetscInt j = 0; j < 3; j++) hess[i][j] = (i == j) ? -PetscSqr(PETSC_PI) * c[i] : 0.;
2909   }
2910 }
2911 
2912 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2913 static PetscErrorCode TPSExtrudeNormalFunc_SchwarzP(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2914 {
2915   for (PetscInt i = 0; i < 3; i++) u[i] = -PETSC_PI * PetscSinReal(x[i] * PETSC_PI);
2916   return PETSC_SUCCESS;
2917 }
2918 
2919 /*
2920  The Gyroid implicit surface is
2921 
2922  f(x,y,z) = sin(pi * x) * cos (pi * (y + 1/2))  + sin(pi * (y + 1/2)) * cos(pi * (z + 1/4)) + sin(pi * (z + 1/4)) * cos(pi * x)
2923 
2924 */
2925 static void TPSEvaluate_Gyroid(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2926 {
2927   PetscReal s[3] = {PetscSinReal(PETSC_PI * y[0]), PetscSinReal(PETSC_PI * (y[1] + .5)), PetscSinReal(PETSC_PI * (y[2] + .25))};
2928   PetscReal c[3] = {PetscCosReal(PETSC_PI * y[0]), PetscCosReal(PETSC_PI * (y[1] + .5)), PetscCosReal(PETSC_PI * (y[2] + .25))};
2929   f[0]           = s[0] * c[1] + s[1] * c[2] + s[2] * c[0];
2930   grad[0]        = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2931   grad[1]        = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2932   grad[2]        = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2933   hess[0][0]     = -PetscSqr(PETSC_PI) * (s[0] * c[1] + s[2] * c[0]);
2934   hess[0][1]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2935   hess[0][2]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2936   hess[1][0]     = -PetscSqr(PETSC_PI) * (s[1] * c[2] + s[0] * c[1]);
2937   hess[1][1]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2938   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2939   hess[2][0]     = -PetscSqr(PETSC_PI) * (s[2] * c[0] + s[1] * c[2]);
2940   hess[2][1]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2941   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2942 }
2943 
2944 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2945 static PetscErrorCode TPSExtrudeNormalFunc_Gyroid(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2946 {
2947   PetscReal s[3] = {PetscSinReal(PETSC_PI * x[0]), PetscSinReal(PETSC_PI * (x[1] + .5)), PetscSinReal(PETSC_PI * (x[2] + .25))};
2948   PetscReal c[3] = {PetscCosReal(PETSC_PI * x[0]), PetscCosReal(PETSC_PI * (x[1] + .5)), PetscCosReal(PETSC_PI * (x[2] + .25))};
2949   u[0]           = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2950   u[1]           = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2951   u[2]           = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2952   return PETSC_SUCCESS;
2953 }
2954 
2955 /*
2956    We wish to solve
2957 
2958          min_y || y - x ||^2  subject to f(y) = 0
2959 
2960    Let g(y) = grad(f).  The minimization problem is equivalent to asking to satisfy
2961    f(y) = 0 and (y-x) is parallel to g(y).  We do this by using Householder QR to obtain a basis for the
2962    tangent space and ask for both components in the tangent space to be zero.
2963 
2964    Take g to be a column vector and compute the "full QR" factorization Q R = g,
2965    where Q = I - 2 n n^T is a symmetric orthogonal matrix.
2966    The first column of Q is parallel to g so the remaining two columns span the null space.
2967    Let Qn = Q[:,1:] be those remaining columns.  Then Qn Qn^T is an orthogonal projector into the tangent space.
2968    Since Q is symmetric, this is equivalent to multiplying by Q and taking the last two entries.
2969    In total, we have a system of 3 equations in 3 unknowns:
2970 
2971      f(y) = 0                       1 equation
2972      Qn^T (y - x) = 0               2 equations
2973 
2974    Here, we compute the residual and Jacobian of this system.
2975 */
2976 static void TPSNearestPointResJac(TPSEvaluateFunc feval, const PetscScalar x[], const PetscScalar y[], PetscScalar res[], PetscScalar J[])
2977 {
2978   PetscReal yreal[3] = {PetscRealPart(y[0]), PetscRealPart(y[1]), PetscRealPart(y[2])};
2979   PetscReal d[3]     = {PetscRealPart(y[0] - x[0]), PetscRealPart(y[1] - x[1]), PetscRealPart(y[2] - x[2])};
2980   PetscReal f, grad[3], n[3], norm, norm_y[3], nd, nd_y[3], sign;
2981   PetscReal n_y[3][3] = {
2982     {0, 0, 0},
2983     {0, 0, 0},
2984     {0, 0, 0}
2985   };
2986 
2987   feval(yreal, &f, grad, n_y);
2988 
2989   for (PetscInt i = 0; i < 3; i++) n[i] = grad[i];
2990   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2991   for (PetscInt i = 0; i < 3; i++) norm_y[i] = 1. / norm * n[i] * n_y[i][i];
2992 
2993   // Define the Householder reflector
2994   sign = n[0] >= 0 ? 1. : -1.;
2995   n[0] += norm * sign;
2996   for (PetscInt i = 0; i < 3; i++) n_y[0][i] += norm_y[i] * sign;
2997 
2998   norm      = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2999   norm_y[0] = 1. / norm * (n[0] * n_y[0][0]);
3000   norm_y[1] = 1. / norm * (n[0] * n_y[0][1] + n[1] * n_y[1][1]);
3001   norm_y[2] = 1. / norm * (n[0] * n_y[0][2] + n[2] * n_y[2][2]);
3002 
3003   for (PetscInt i = 0; i < 3; i++) {
3004     n[i] /= norm;
3005     for (PetscInt j = 0; j < 3; j++) {
3006       // note that n[i] is n_old[i]/norm when executing the code below
3007       n_y[i][j] = n_y[i][j] / norm - n[i] / norm * norm_y[j];
3008     }
3009   }
3010 
3011   nd = n[0] * d[0] + n[1] * d[1] + n[2] * d[2];
3012   for (PetscInt i = 0; i < 3; i++) nd_y[i] = n[i] + n_y[0][i] * d[0] + n_y[1][i] * d[1] + n_y[2][i] * d[2];
3013 
3014   res[0] = f;
3015   res[1] = d[1] - 2 * n[1] * nd;
3016   res[2] = d[2] - 2 * n[2] * nd;
3017   // J[j][i] is J_{ij} (column major)
3018   for (PetscInt j = 0; j < 3; j++) {
3019     J[0 + j * 3] = grad[j];
3020     J[1 + j * 3] = (j == 1) * 1. - 2 * (n_y[1][j] * nd + n[1] * nd_y[j]);
3021     J[2 + j * 3] = (j == 2) * 1. - 2 * (n_y[2][j] * nd + n[2] * nd_y[j]);
3022   }
3023 }
3024 
3025 /*
3026    Project x to the nearest point on the implicit surface using Newton's method.
3027 */
3028 static PetscErrorCode TPSNearestPoint(TPSEvaluateFunc feval, PetscScalar x[])
3029 {
3030   PetscScalar y[3] = {x[0], x[1], x[2]}; // Initial guess
3031 
3032   PetscFunctionBegin;
3033   for (PetscInt iter = 0; iter < 10; iter++) {
3034     PetscScalar res[3], J[9];
3035     PetscReal   resnorm;
3036     TPSNearestPointResJac(feval, x, y, res, J);
3037     resnorm = PetscSqrtReal(PetscSqr(PetscRealPart(res[0])) + PetscSqr(PetscRealPart(res[1])) + PetscSqr(PetscRealPart(res[2])));
3038     if (0) { // Turn on this monitor if you need to confirm quadratic convergence
3039       PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%" PetscInt_FMT "] res [%g %g %g]\n", iter, (double)PetscRealPart(res[0]), (double)PetscRealPart(res[1]), (double)PetscRealPart(res[2])));
3040     }
3041     if (resnorm < PETSC_SMALL) break;
3042 
3043     // Take the Newton step
3044     PetscCall(PetscKernel_A_gets_inverse_A_3(J, 0., PETSC_FALSE, NULL));
3045     PetscKernel_v_gets_v_minus_A_times_w_3(y, J, res);
3046   }
3047   for (PetscInt i = 0; i < 3; i++) x[i] = y[i];
3048   PetscFunctionReturn(PETSC_SUCCESS);
3049 }
3050 
3051 const char *const DMPlexTPSTypes[] = {"SCHWARZ_P", "GYROID", "DMPlexTPSType", "DMPLEX_TPS_", NULL};
3052 
3053 static PetscErrorCode DMPlexCreateTPSMesh_Internal(DM dm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness)
3054 {
3055   PetscMPIInt rank;
3056   PetscInt    topoDim = 2, spaceDim = 3, numFaces = 0, numVertices = 0, numEdges = 0;
3057   PetscInt(*edges)[2] = NULL, *edgeSets = NULL;
3058   PetscInt           *cells_flat = NULL;
3059   PetscReal          *vtxCoords  = NULL;
3060   TPSEvaluateFunc     evalFunc   = NULL;
3061   PetscSimplePointFn *normalFunc = NULL;
3062   DMLabel             label;
3063 
3064   PetscFunctionBegin;
3065   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
3066   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3067   PetscCheck((layers != 0) ^ (thickness == 0.), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Layers %" PetscInt_FMT " must be nonzero iff thickness %g is nonzero", layers, (double)thickness);
3068   switch (tpstype) {
3069   case DMPLEX_TPS_SCHWARZ_P:
3070     PetscCheck(!periodic || (periodic[0] == DM_BOUNDARY_NONE && periodic[1] == DM_BOUNDARY_NONE && periodic[2] == DM_BOUNDARY_NONE), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Schwarz P does not support periodic meshes");
3071     if (rank == 0) {
3072       PetscInt(*cells)[6][4][4] = NULL; // [junction, junction-face, cell, conn]
3073       PetscInt  Njunctions = 0, Ncuts = 0, Npipes[3], vcount;
3074       PetscReal L = 1;
3075 
3076       Npipes[0]   = (extent[0] + 1) * extent[1] * extent[2];
3077       Npipes[1]   = extent[0] * (extent[1] + 1) * extent[2];
3078       Npipes[2]   = extent[0] * extent[1] * (extent[2] + 1);
3079       Njunctions  = extent[0] * extent[1] * extent[2];
3080       Ncuts       = 2 * (extent[0] * extent[1] + extent[1] * extent[2] + extent[2] * extent[0]);
3081       numVertices = 4 * (Npipes[0] + Npipes[1] + Npipes[2]) + 8 * Njunctions;
3082       PetscCall(PetscMalloc1(3 * numVertices, &vtxCoords));
3083       PetscCall(PetscMalloc1(Njunctions, &cells));
3084       PetscCall(PetscMalloc1(Ncuts * 4, &edges));
3085       PetscCall(PetscMalloc1(Ncuts * 4, &edgeSets));
3086       // x-normal pipes
3087       vcount = 0;
3088       for (PetscInt i = 0; i < extent[0] + 1; i++) {
3089         for (PetscInt j = 0; j < extent[1]; j++) {
3090           for (PetscInt k = 0; k < extent[2]; k++) {
3091             for (PetscInt l = 0; l < 4; l++) {
3092               vtxCoords[vcount++] = (2 * i - 1) * L;
3093               vtxCoords[vcount++] = 2 * j * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3094               vtxCoords[vcount++] = 2 * k * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3095             }
3096           }
3097         }
3098       }
3099       // y-normal pipes
3100       for (PetscInt i = 0; i < extent[0]; i++) {
3101         for (PetscInt j = 0; j < extent[1] + 1; j++) {
3102           for (PetscInt k = 0; k < extent[2]; k++) {
3103             for (PetscInt l = 0; l < 4; l++) {
3104               vtxCoords[vcount++] = 2 * i * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3105               vtxCoords[vcount++] = (2 * j - 1) * L;
3106               vtxCoords[vcount++] = 2 * k * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3107             }
3108           }
3109         }
3110       }
3111       // z-normal pipes
3112       for (PetscInt i = 0; i < extent[0]; i++) {
3113         for (PetscInt j = 0; j < extent[1]; j++) {
3114           for (PetscInt k = 0; k < extent[2] + 1; k++) {
3115             for (PetscInt l = 0; l < 4; l++) {
3116               vtxCoords[vcount++] = 2 * i * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3117               vtxCoords[vcount++] = 2 * j * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3118               vtxCoords[vcount++] = (2 * k - 1) * L;
3119             }
3120           }
3121         }
3122       }
3123       // junctions
3124       for (PetscInt i = 0; i < extent[0]; i++) {
3125         for (PetscInt j = 0; j < extent[1]; j++) {
3126           for (PetscInt k = 0; k < extent[2]; k++) {
3127             const PetscInt J = (i * extent[1] + j) * extent[2] + k, Jvoff = (Npipes[0] + Npipes[1] + Npipes[2]) * 4 + J * 8;
3128             PetscCheck(vcount / 3 == Jvoff, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected vertex count");
3129             for (PetscInt ii = 0; ii < 2; ii++) {
3130               for (PetscInt jj = 0; jj < 2; jj++) {
3131                 for (PetscInt kk = 0; kk < 2; kk++) {
3132                   double Ls           = (1 - sqrt(2) / 4) * L;
3133                   vtxCoords[vcount++] = 2 * i * L + (2 * ii - 1) * Ls;
3134                   vtxCoords[vcount++] = 2 * j * L + (2 * jj - 1) * Ls;
3135                   vtxCoords[vcount++] = 2 * k * L + (2 * kk - 1) * Ls;
3136                 }
3137               }
3138             }
3139             const PetscInt jfaces[3][2][4] = {
3140               {{3, 1, 0, 2}, {7, 5, 4, 6}}, // x-aligned
3141               {{5, 4, 0, 1}, {7, 6, 2, 3}}, // y-aligned
3142               {{6, 2, 0, 4}, {7, 3, 1, 5}}  // z-aligned
3143             };
3144             const PetscInt pipe_lo[3] = {// vertex numbers of pipes
3145                                          ((i * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + Npipes[0] + Npipes[1]) * 4};
3146             const PetscInt pipe_hi[3] = {// vertex numbers of pipes
3147                                          (((i + 1) * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j + 1) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + 1 + Npipes[0] + Npipes[1]) * 4};
3148             for (PetscInt dir = 0; dir < 3; dir++) { // x,y,z
3149               const PetscInt ijk[3] = {i, j, k};
3150               for (PetscInt l = 0; l < 4; l++) { // rotations
3151                 cells[J][dir * 2 + 0][l][0] = pipe_lo[dir] + l;
3152                 cells[J][dir * 2 + 0][l][1] = Jvoff + jfaces[dir][0][l];
3153                 cells[J][dir * 2 + 0][l][2] = Jvoff + jfaces[dir][0][(l - 1 + 4) % 4];
3154                 cells[J][dir * 2 + 0][l][3] = pipe_lo[dir] + (l - 1 + 4) % 4;
3155                 cells[J][dir * 2 + 1][l][0] = Jvoff + jfaces[dir][1][l];
3156                 cells[J][dir * 2 + 1][l][1] = pipe_hi[dir] + l;
3157                 cells[J][dir * 2 + 1][l][2] = pipe_hi[dir] + (l - 1 + 4) % 4;
3158                 cells[J][dir * 2 + 1][l][3] = Jvoff + jfaces[dir][1][(l - 1 + 4) % 4];
3159                 if (ijk[dir] == 0) {
3160                   edges[numEdges][0] = pipe_lo[dir] + l;
3161                   edges[numEdges][1] = pipe_lo[dir] + (l + 1) % 4;
3162                   edgeSets[numEdges] = dir * 2 + 1;
3163                   numEdges++;
3164                 }
3165                 if (ijk[dir] + 1 == extent[dir]) {
3166                   edges[numEdges][0] = pipe_hi[dir] + l;
3167                   edges[numEdges][1] = pipe_hi[dir] + (l + 1) % 4;
3168                   edgeSets[numEdges] = dir * 2 + 2;
3169                   numEdges++;
3170                 }
3171               }
3172             }
3173           }
3174         }
3175       }
3176       PetscCheck(numEdges == Ncuts * 4, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Edge count %" PetscInt_FMT " incompatible with number of cuts %" PetscInt_FMT, numEdges, Ncuts);
3177       numFaces   = 24 * Njunctions;
3178       cells_flat = cells[0][0][0];
3179     }
3180     evalFunc   = TPSEvaluate_SchwarzP;
3181     normalFunc = TPSExtrudeNormalFunc_SchwarzP;
3182     break;
3183   case DMPLEX_TPS_GYROID:
3184     if (rank == 0) {
3185       // This is a coarse mesh approximation of the gyroid shifted to being the zero of the level set
3186       //
3187       //     sin(pi*x)*cos(pi*(y+1/2)) + sin(pi*(y+1/2))*cos(pi*(z+1/4)) + sin(pi*(z+1/4))*cos(x)
3188       //
3189       // on the cell [0,2]^3.
3190       //
3191       // Think about dividing that cell into four columns, and focus on the column [0,1]x[0,1]x[0,2].
3192       // If you looked at the gyroid in that column at different slices of z you would see that it kind of spins
3193       // like a boomerang:
3194       //
3195       //     z = 0          z = 1/4        z = 1/2        z = 3/4     //
3196       //     -----          -------        -------        -------     //
3197       //                                                              //
3198       //     +       +      +       +      +       +      +   \   +   //
3199       //      \                                   /            \      //
3200       //       \            `-_   _-'            /              }     //
3201       //        *-_            `-'            _-'              /      //
3202       //     +     `-+      +       +      +-'     +      +   /   +   //
3203       //                                                              //
3204       //                                                              //
3205       //     z = 1          z = 5/4        z = 3/2        z = 7/4     //
3206       //     -----          -------        -------        -------     //
3207       //                                                              //
3208       //     +-_     +      +       +      +     _-+      +   /   +   //
3209       //        `-_            _-_            _-`            /        //
3210       //           \        _-'   `-_        /              {         //
3211       //            \                       /                \        //
3212       //     +       +      +       +      +       +      +   \   +   //
3213       //
3214       //
3215       // This course mesh approximates each of these slices by two line segments,
3216       // and then connects the segments in consecutive layers with quadrilateral faces.
3217       // All of the end points of the segments are multiples of 1/4 except for the
3218       // point * in the picture for z = 0 above and the similar points in other layers.
3219       // That point is at (gamma, gamma, 0), where gamma is calculated below.
3220       //
3221       // The column  [1,2]x[1,2]x[0,2] looks the same as this column;
3222       // The columns [1,2]x[0,1]x[0,2] and [0,1]x[1,2]x[0,2] are mirror images.
3223       //
3224       // As for how this method turned into the names given to the vertices:
3225       // that was not systematic, it was just the way it worked out in my handwritten notes.
3226 
3227       PetscInt facesPerBlock = 64;
3228       PetscInt vertsPerBlock = 56;
3229       PetscInt extentPlus[3];
3230       PetscInt numBlocks, numBlocksPlus;
3231       const PetscInt A = 0, B = 1, C = 2, D = 3, E = 4, F = 5, G = 6, H = 7, II = 8, J = 9, K = 10, L = 11, M = 12, N = 13, O = 14, P = 15, Q = 16, R = 17, S = 18, T = 19, U = 20, V = 21, W = 22, X = 23, Y = 24, Z = 25, Ap = 26, Bp = 27, Cp = 28, Dp = 29, Ep = 30, Fp = 31, Gp = 32, Hp = 33, Ip = 34, Jp = 35, Kp = 36, Lp = 37, Mp = 38, Np = 39, Op = 40, Pp = 41, Qp = 42, Rp = 43, Sp = 44, Tp = 45, Up = 46, Vp = 47, Wp = 48, Xp = 49, Yp = 50, Zp = 51, Aq = 52, Bq = 53, Cq = 54, Dq = 55;
3232       const PetscInt pattern[64][4] = {
3233         /* face to vertex within the coarse discretization of a single gyroid block */
3234         /* layer 0 */
3235         {A,           C,           K,           G          },
3236         {C,           B,           II,          K          },
3237         {D,           A,           H,           L          },
3238         {B + 56 * 1,  D,           L,           J          },
3239         {E,           B + 56 * 1,  J,           N          },
3240         {A + 56 * 2,  E,           N,           H + 56 * 2 },
3241         {F,           A + 56 * 2,  G + 56 * 2,  M          },
3242         {B,           F,           M,           II         },
3243         /* layer 1 */
3244         {G,           K,           Q,           O          },
3245         {K,           II,          P,           Q          },
3246         {L,           H,           O + 56 * 1,  R          },
3247         {J,           L,           R,           P          },
3248         {N,           J,           P,           S          },
3249         {H + 56 * 2,  N,           S,           O + 56 * 3 },
3250         {M,           G + 56 * 2,  O + 56 * 2,  T          },
3251         {II,          M,           T,           P          },
3252         /* layer 2 */
3253         {O,           Q,           Y,           U          },
3254         {Q,           P,           W,           Y          },
3255         {R,           O + 56 * 1,  U + 56 * 1,  Ap         },
3256         {P,           R,           Ap,          W          },
3257         {S,           P,           X,           Bp         },
3258         {O + 56 * 3,  S,           Bp,          V + 56 * 1 },
3259         {T,           O + 56 * 2,  V,           Z          },
3260         {P,           T,           Z,           X          },
3261         /* layer 3 */
3262         {U,           Y,           Ep,          Dp         },
3263         {Y,           W,           Cp,          Ep         },
3264         {Ap,          U + 56 * 1,  Dp + 56 * 1, Gp         },
3265         {W,           Ap,          Gp,          Cp         },
3266         {Bp,          X,           Cp + 56 * 2, Fp         },
3267         {V + 56 * 1,  Bp,          Fp,          Dp + 56 * 1},
3268         {Z,           V,           Dp,          Hp         },
3269         {X,           Z,           Hp,          Cp + 56 * 2},
3270         /* layer 4 */
3271         {Dp,          Ep,          Mp,          Kp         },
3272         {Ep,          Cp,          Ip,          Mp         },
3273         {Gp,          Dp + 56 * 1, Lp,          Np         },
3274         {Cp,          Gp,          Np,          Jp         },
3275         {Fp,          Cp + 56 * 2, Jp + 56 * 2, Pp         },
3276         {Dp + 56 * 1, Fp,          Pp,          Lp         },
3277         {Hp,          Dp,          Kp,          Op         },
3278         {Cp + 56 * 2, Hp,          Op,          Ip + 56 * 2},
3279         /* layer 5 */
3280         {Kp,          Mp,          Sp,          Rp         },
3281         {Mp,          Ip,          Qp,          Sp         },
3282         {Np,          Lp,          Rp,          Tp         },
3283         {Jp,          Np,          Tp,          Qp + 56 * 1},
3284         {Pp,          Jp + 56 * 2, Qp + 56 * 3, Up         },
3285         {Lp,          Pp,          Up,          Rp         },
3286         {Op,          Kp,          Rp,          Vp         },
3287         {Ip + 56 * 2, Op,          Vp,          Qp + 56 * 2},
3288         /* layer 6 */
3289         {Rp,          Sp,          Aq,          Yp         },
3290         {Sp,          Qp,          Wp,          Aq         },
3291         {Tp,          Rp,          Yp,          Cq         },
3292         {Qp + 56 * 1, Tp,          Cq,          Wp + 56 * 1},
3293         {Up,          Qp + 56 * 3, Xp + 56 * 1, Dq         },
3294         {Rp,          Up,          Dq,          Zp         },
3295         {Vp,          Rp,          Zp,          Bq         },
3296         {Qp + 56 * 2, Vp,          Bq,          Xp         },
3297         /* layer 7 (the top is the periodic image of the bottom of layer 0) */
3298         {Yp,          Aq,          C + 56 * 4,  A + 56 * 4 },
3299         {Aq,          Wp,          B + 56 * 4,  C + 56 * 4 },
3300         {Cq,          Yp,          A + 56 * 4,  D + 56 * 4 },
3301         {Wp + 56 * 1, Cq,          D + 56 * 4,  B + 56 * 5 },
3302         {Dq,          Xp + 56 * 1, B + 56 * 5,  E + 56 * 4 },
3303         {Zp,          Dq,          E + 56 * 4,  A + 56 * 6 },
3304         {Bq,          Zp,          A + 56 * 6,  F + 56 * 4 },
3305         {Xp,          Bq,          F + 56 * 4,  B + 56 * 4 }
3306       };
3307       const PetscReal gamma                = PetscAcosReal((PetscSqrtReal(3.) - 1.) / PetscSqrtReal(2.)) / PETSC_PI;
3308       const PetscReal patternCoords[56][3] = {
3309         {1.,        0.,        0.  }, /* A  */
3310         {0.,        1.,        0.  }, /* B  */
3311         {gamma,     gamma,     0.  }, /* C  */
3312         {1 + gamma, 1 - gamma, 0.  }, /* D  */
3313         {2 - gamma, 2 - gamma, 0.  }, /* E  */
3314         {1 - gamma, 1 + gamma, 0.  }, /* F  */
3315 
3316         {.5,        0,         .25 }, /* G  */
3317         {1.5,       0.,        .25 }, /* H  */
3318         {.5,        1.,        .25 }, /* II */
3319         {1.5,       1.,        .25 }, /* J  */
3320         {.25,       .5,        .25 }, /* K  */
3321         {1.25,      .5,        .25 }, /* L  */
3322         {.75,       1.5,       .25 }, /* M  */
3323         {1.75,      1.5,       .25 }, /* N  */
3324 
3325         {0.,        0.,        .5  }, /* O  */
3326         {1.,        1.,        .5  }, /* P  */
3327         {gamma,     1 - gamma, .5  }, /* Q  */
3328         {1 + gamma, gamma,     .5  }, /* R  */
3329         {2 - gamma, 1 + gamma, .5  }, /* S  */
3330         {1 - gamma, 2 - gamma, .5  }, /* T  */
3331 
3332         {0.,        .5,        .75 }, /* U  */
3333         {0.,        1.5,       .75 }, /* V  */
3334         {1.,        .5,        .75 }, /* W  */
3335         {1.,        1.5,       .75 }, /* X  */
3336         {.5,        .75,       .75 }, /* Y  */
3337         {.5,        1.75,      .75 }, /* Z  */
3338         {1.5,       .25,       .75 }, /* Ap */
3339         {1.5,       1.25,      .75 }, /* Bp */
3340 
3341         {1.,        0.,        1.  }, /* Cp */
3342         {0.,        1.,        1.  }, /* Dp */
3343         {1 - gamma, 1 - gamma, 1.  }, /* Ep */
3344         {1 + gamma, 1 + gamma, 1.  }, /* Fp */
3345         {2 - gamma, gamma,     1.  }, /* Gp */
3346         {gamma,     2 - gamma, 1.  }, /* Hp */
3347 
3348         {.5,        0.,        1.25}, /* Ip */
3349         {1.5,       0.,        1.25}, /* Jp */
3350         {.5,        1.,        1.25}, /* Kp */
3351         {1.5,       1.,        1.25}, /* Lp */
3352         {.75,       .5,        1.25}, /* Mp */
3353         {1.75,      .5,        1.25}, /* Np */
3354         {.25,       1.5,       1.25}, /* Op */
3355         {1.25,      1.5,       1.25}, /* Pp */
3356 
3357         {0.,        0.,        1.5 }, /* Qp */
3358         {1.,        1.,        1.5 }, /* Rp */
3359         {1 - gamma, gamma,     1.5 }, /* Sp */
3360         {2 - gamma, 1 - gamma, 1.5 }, /* Tp */
3361         {1 + gamma, 2 - gamma, 1.5 }, /* Up */
3362         {gamma,     1 + gamma, 1.5 }, /* Vp */
3363 
3364         {0.,        .5,        1.75}, /* Wp */
3365         {0.,        1.5,       1.75}, /* Xp */
3366         {1.,        .5,        1.75}, /* Yp */
3367         {1.,        1.5,       1.75}, /* Zp */
3368         {.5,        .25,       1.75}, /* Aq */
3369         {.5,        1.25,      1.75}, /* Bq */
3370         {1.5,       .75,       1.75}, /* Cq */
3371         {1.5,       1.75,      1.75}, /* Dq */
3372       };
3373       PetscInt(*cells)[64][4] = NULL;
3374       PetscBool *seen;
3375       PetscInt  *vertToTrueVert;
3376       PetscInt   count;
3377 
3378       for (PetscInt i = 0; i < 3; i++) extentPlus[i] = extent[i] + 1;
3379       numBlocks = 1;
3380       for (PetscInt i = 0; i < 3; i++) numBlocks *= extent[i];
3381       numBlocksPlus = 1;
3382       for (PetscInt i = 0; i < 3; i++) numBlocksPlus *= extentPlus[i];
3383       numFaces = numBlocks * facesPerBlock;
3384       PetscCall(PetscMalloc1(numBlocks, &cells));
3385       PetscCall(PetscCalloc1(numBlocksPlus * vertsPerBlock, &seen));
3386       for (PetscInt k = 0; k < extent[2]; k++) {
3387         for (PetscInt j = 0; j < extent[1]; j++) {
3388           for (PetscInt i = 0; i < extent[0]; i++) {
3389             for (PetscInt f = 0; f < facesPerBlock; f++) {
3390               for (PetscInt v = 0; v < 4; v++) {
3391                 PetscInt vertRaw     = pattern[f][v];
3392                 PetscInt blockidx    = vertRaw / 56;
3393                 PetscInt patternvert = vertRaw % 56;
3394                 PetscInt xplus       = (blockidx & 1);
3395                 PetscInt yplus       = (blockidx & 2) >> 1;
3396                 PetscInt zplus       = (blockidx & 4) >> 2;
3397                 PetscInt zcoord      = (periodic && periodic[2] == DM_BOUNDARY_PERIODIC) ? ((k + zplus) % extent[2]) : (k + zplus);
3398                 PetscInt ycoord      = (periodic && periodic[1] == DM_BOUNDARY_PERIODIC) ? ((j + yplus) % extent[1]) : (j + yplus);
3399                 PetscInt xcoord      = (periodic && periodic[0] == DM_BOUNDARY_PERIODIC) ? ((i + xplus) % extent[0]) : (i + xplus);
3400                 PetscInt vert        = ((zcoord * extentPlus[1] + ycoord) * extentPlus[0] + xcoord) * 56 + patternvert;
3401 
3402                 cells[(k * extent[1] + j) * extent[0] + i][f][v] = vert;
3403                 seen[vert]                                       = PETSC_TRUE;
3404               }
3405             }
3406           }
3407         }
3408       }
3409       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++)
3410         if (seen[i]) numVertices++;
3411       count = 0;
3412       PetscCall(PetscMalloc1(numBlocksPlus * vertsPerBlock, &vertToTrueVert));
3413       PetscCall(PetscMalloc1(numVertices * 3, &vtxCoords));
3414       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) vertToTrueVert[i] = -1;
3415       for (PetscInt k = 0; k < extentPlus[2]; k++) {
3416         for (PetscInt j = 0; j < extentPlus[1]; j++) {
3417           for (PetscInt i = 0; i < extentPlus[0]; i++) {
3418             for (PetscInt v = 0; v < vertsPerBlock; v++) {
3419               PetscInt vIdx = ((k * extentPlus[1] + j) * extentPlus[0] + i) * vertsPerBlock + v;
3420 
3421               if (seen[vIdx]) {
3422                 PetscInt thisVert;
3423 
3424                 vertToTrueVert[vIdx] = thisVert = count++;
3425 
3426                 for (PetscInt d = 0; d < 3; d++) vtxCoords[3 * thisVert + d] = patternCoords[v][d];
3427                 vtxCoords[3 * thisVert + 0] += i * 2;
3428                 vtxCoords[3 * thisVert + 1] += j * 2;
3429                 vtxCoords[3 * thisVert + 2] += k * 2;
3430               }
3431             }
3432           }
3433         }
3434       }
3435       for (PetscInt i = 0; i < numBlocks; i++) {
3436         for (PetscInt f = 0; f < facesPerBlock; f++) {
3437           for (PetscInt v = 0; v < 4; v++) cells[i][f][v] = vertToTrueVert[cells[i][f][v]];
3438         }
3439       }
3440       PetscCall(PetscFree(vertToTrueVert));
3441       PetscCall(PetscFree(seen));
3442       cells_flat = cells[0][0];
3443       numEdges   = 0;
3444       for (PetscInt i = 0; i < numFaces; i++) {
3445         for (PetscInt e = 0; e < 4; e++) {
3446           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3447           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3448 
3449           for (PetscInt d = 0; d < 3; d++) {
3450             if (!periodic || periodic[0] != DM_BOUNDARY_PERIODIC) {
3451               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) numEdges++;
3452               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) numEdges++;
3453             }
3454           }
3455         }
3456       }
3457       PetscCall(PetscMalloc1(numEdges, &edges));
3458       PetscCall(PetscMalloc1(numEdges, &edgeSets));
3459       for (PetscInt edge = 0, i = 0; i < numFaces; i++) {
3460         for (PetscInt e = 0; e < 4; e++) {
3461           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3462           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3463 
3464           for (PetscInt d = 0; d < 3; d++) {
3465             if (!periodic || periodic[d] != DM_BOUNDARY_PERIODIC) {
3466               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) {
3467                 edges[edge][0]   = ev[0];
3468                 edges[edge][1]   = ev[1];
3469                 edgeSets[edge++] = 2 * d;
3470               }
3471               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) {
3472                 edges[edge][0]   = ev[0];
3473                 edges[edge][1]   = ev[1];
3474                 edgeSets[edge++] = 2 * d + 1;
3475               }
3476             }
3477           }
3478         }
3479       }
3480     }
3481     evalFunc   = TPSEvaluate_Gyroid;
3482     normalFunc = TPSExtrudeNormalFunc_Gyroid;
3483     break;
3484   }
3485 
3486   PetscCall(DMSetDimension(dm, topoDim));
3487   if (rank == 0) PetscCall(DMPlexBuildFromCellList(dm, numFaces, numVertices, 4, cells_flat));
3488   else PetscCall(DMPlexBuildFromCellList(dm, 0, 0, 0, NULL));
3489   PetscCall(PetscFree(cells_flat));
3490   {
3491     DM idm;
3492     PetscCall(DMPlexInterpolate(dm, &idm));
3493     PetscCall(DMPlexReplace_Internal(dm, &idm));
3494   }
3495   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, vtxCoords));
3496   else PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, NULL));
3497   PetscCall(PetscFree(vtxCoords));
3498 
3499   PetscCall(DMCreateLabel(dm, "Face Sets"));
3500   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3501   for (PetscInt e = 0; e < numEdges; e++) {
3502     PetscInt        njoin;
3503     const PetscInt *join, verts[] = {numFaces + edges[e][0], numFaces + edges[e][1]};
3504     PetscCall(DMPlexGetJoin(dm, 2, verts, &njoin, &join));
3505     PetscCheck(njoin == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected unique join of vertices %" PetscInt_FMT " and %" PetscInt_FMT, edges[e][0], edges[e][1]);
3506     PetscCall(DMLabelSetValue(label, join[0], edgeSets[e]));
3507     PetscCall(DMPlexRestoreJoin(dm, 2, verts, &njoin, &join));
3508   }
3509   PetscCall(PetscFree(edges));
3510   PetscCall(PetscFree(edgeSets));
3511   if (tps_distribute) {
3512     DM               pdm = NULL;
3513     PetscPartitioner part;
3514 
3515     PetscCall(DMPlexGetPartitioner(dm, &part));
3516     PetscCall(PetscPartitionerSetFromOptions(part));
3517     PetscCall(DMPlexDistribute(dm, 0, NULL, &pdm));
3518     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
3519     // Do not auto-distribute again
3520     PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
3521   }
3522 
3523   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3524   for (PetscInt refine = 0; refine < refinements; refine++) {
3525     PetscInt     m;
3526     DM           dmf;
3527     Vec          X;
3528     PetscScalar *x;
3529     PetscCall(DMRefine(dm, MPI_COMM_NULL, &dmf));
3530     PetscCall(DMPlexReplace_Internal(dm, &dmf));
3531 
3532     PetscCall(DMGetCoordinatesLocal(dm, &X));
3533     PetscCall(VecGetLocalSize(X, &m));
3534     PetscCall(VecGetArray(X, &x));
3535     for (PetscInt i = 0; i < m; i += 3) PetscCall(TPSNearestPoint(evalFunc, &x[i]));
3536     PetscCall(VecRestoreArray(X, &x));
3537   }
3538 
3539   // Face Sets has already been propagated to new vertices during refinement; this propagates to the initial vertices.
3540   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3541   PetscCall(DMPlexLabelComplete(dm, label));
3542 
3543   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
3544 
3545   if (thickness > 0) {
3546     DM              edm, cdm, ecdm;
3547     DMPlexTransform tr;
3548     const char     *prefix;
3549     PetscOptions    options;
3550     // Code from DMPlexExtrude
3551     PetscCall(DMPlexTransformCreate(PetscObjectComm((PetscObject)dm), &tr));
3552     PetscCall(DMPlexTransformSetDM(tr, dm));
3553     PetscCall(DMPlexTransformSetType(tr, DMPLEXEXTRUDE));
3554     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
3555     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)tr, prefix));
3556     PetscCall(PetscObjectGetOptions((PetscObject)dm, &options));
3557     PetscCall(PetscObjectSetOptions((PetscObject)tr, options));
3558     PetscCall(DMPlexTransformExtrudeSetLayers(tr, layers));
3559     PetscCall(DMPlexTransformExtrudeSetThickness(tr, thickness));
3560     PetscCall(DMPlexTransformExtrudeSetTensor(tr, PETSC_FALSE));
3561     PetscCall(DMPlexTransformExtrudeSetSymmetric(tr, PETSC_TRUE));
3562     PetscCall(DMPlexTransformExtrudeSetNormalFunction(tr, normalFunc));
3563     PetscCall(DMPlexTransformSetFromOptions(tr));
3564     PetscCall(PetscObjectSetOptions((PetscObject)tr, NULL));
3565     PetscCall(DMPlexTransformSetUp(tr));
3566     PetscCall(PetscObjectViewFromOptions((PetscObject)tr, NULL, "-dm_plex_tps_transform_view"));
3567     PetscCall(DMPlexTransformApply(tr, dm, &edm));
3568     PetscCall(DMCopyDisc(dm, edm));
3569     PetscCall(DMGetCoordinateDM(dm, &cdm));
3570     PetscCall(DMGetCoordinateDM(edm, &ecdm));
3571     PetscCall(DMCopyDisc(cdm, ecdm));
3572     PetscCall(DMPlexTransformCreateDiscLabels(tr, edm));
3573     PetscCall(DMPlexTransformDestroy(&tr));
3574     if (edm) {
3575       ((DM_Plex *)edm->data)->printFEM    = ((DM_Plex *)dm->data)->printFEM;
3576       ((DM_Plex *)edm->data)->printL2     = ((DM_Plex *)dm->data)->printL2;
3577       ((DM_Plex *)edm->data)->printLocate = ((DM_Plex *)dm->data)->printLocate;
3578     }
3579     PetscCall(DMPlexReplace_Internal(dm, &edm));
3580   }
3581   PetscFunctionReturn(PETSC_SUCCESS);
3582 }
3583 
3584 /*@
3585   DMPlexCreateTPSMesh - Create a distributed, interpolated mesh of a triply-periodic surface
3586 
3587   Collective
3588 
3589   Input Parameters:
3590 + comm           - The communicator for the `DM` object
3591 . tpstype        - Type of triply-periodic surface
3592 . extent         - Array of length 3 containing number of periods in each direction
3593 . periodic       - array of length 3 with periodicity, or `NULL` for non-periodic
3594 . tps_distribute - Distribute 2D manifold mesh prior to refinement and extrusion (more scalable)
3595 . refinements    - Number of factor-of-2 refinements of 2D manifold mesh
3596 . layers         - Number of cell layers extruded in normal direction
3597 - thickness      - Thickness in normal direction
3598 
3599   Output Parameter:
3600 . dm - The `DM` object
3601 
3602   Level: beginner
3603 
3604   Notes:
3605   This meshes the surface of the Schwarz P or Gyroid surfaces.  Schwarz P is the simplest member of the triply-periodic minimal surfaces.
3606   <https://en.wikipedia.org/wiki/Schwarz_minimal_surface#Schwarz_P_(%22Primitive%22)> and can be cut with "clean" boundaries.
3607   The Gyroid <https://en.wikipedia.org/wiki/Gyroid> is another triply-periodic minimal surface with applications in additive manufacturing; it is much more difficult to "cut" since there are no planes of symmetry.
3608   Our implementation creates a very coarse mesh of the surface and refines (by 4-way splitting) as many times as requested.
3609   On each refinement, all vertices are projected to their nearest point on the surface.
3610   This projection could readily be extended to related surfaces.
3611 
3612   See {cite}`maskery2018insights`
3613 
3614   The face (edge) sets for the Schwarz P surface are numbered $1(-x), 2(+x), 3(-y), 4(+y), 5(-z), 6(+z)$.
3615   When the mesh is refined, "Face Sets" contain the new vertices (created during refinement).
3616   Use `DMPlexLabelComplete()` to propagate to coarse-level vertices.
3617 
3618   Developer Notes:
3619   The Gyroid mesh does not currently mark boundary sets.
3620 
3621 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMSetType()`, `DMCreate()`
3622 @*/
3623 PetscErrorCode DMPlexCreateTPSMesh(MPI_Comm comm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness, DM *dm)
3624 {
3625   PetscFunctionBegin;
3626   PetscCall(DMCreate(comm, dm));
3627   PetscCall(DMSetType(*dm, DMPLEX));
3628   PetscCall(DMPlexCreateTPSMesh_Internal(*dm, tpstype, extent, periodic, tps_distribute, refinements, layers, thickness));
3629   PetscFunctionReturn(PETSC_SUCCESS);
3630 }
3631 
3632 /*@
3633   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
3634 
3635   Collective
3636 
3637   Input Parameters:
3638 + comm    - The communicator for the `DM` object
3639 . dim     - The dimension
3640 . simplex - Use simplices, or tensor product cells
3641 - R       - The radius
3642 
3643   Output Parameter:
3644 . dm - The `DM` object
3645 
3646   Level: beginner
3647 
3648 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBallMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3649 @*/
3650 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscReal R, DM *dm)
3651 {
3652   PetscFunctionBegin;
3653   PetscAssertPointer(dm, 5);
3654   PetscCall(DMCreate(comm, dm));
3655   PetscCall(DMSetType(*dm, DMPLEX));
3656   PetscCall(DMPlexCreateSphereMesh_Internal(*dm, dim, simplex, R));
3657   PetscFunctionReturn(PETSC_SUCCESS);
3658 }
3659 
3660 static PetscErrorCode DMPlexCreateBallMesh_Internal(DM dm, PetscInt dim, PetscReal R)
3661 {
3662   DM          sdm, vol;
3663   DMLabel     bdlabel;
3664   const char *prefix;
3665 
3666   PetscFunctionBegin;
3667   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
3668   PetscCall(DMSetType(sdm, DMPLEX));
3669   PetscCall(DMGetOptionsPrefix(dm, &prefix));
3670   PetscCall(DMSetOptionsPrefix(sdm, prefix));
3671   PetscCall(DMAppendOptionsPrefix(sdm, "bd_"));
3672   PetscCall(DMPlexDistributeSetDefault(sdm, PETSC_FALSE));
3673   PetscCall(DMPlexCreateSphereMesh_Internal(sdm, dim - 1, PETSC_TRUE, R));
3674   PetscCall(DMSetFromOptions(sdm));
3675   PetscCall(DMViewFromOptions(sdm, NULL, "-dm_view"));
3676   PetscCall(DMPlexGenerate(sdm, NULL, PETSC_TRUE, &vol));
3677   PetscCall(DMDestroy(&sdm));
3678   PetscCall(DMPlexReplace_Internal(dm, &vol));
3679   PetscCall(DMCreateLabel(dm, "marker"));
3680   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
3681   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
3682   PetscCall(DMPlexLabelComplete(dm, bdlabel));
3683   PetscFunctionReturn(PETSC_SUCCESS);
3684 }
3685 
3686 /*@
3687   DMPlexCreateBallMesh - Creates a simplex mesh on the d-dimensional ball, B^d.
3688 
3689   Collective
3690 
3691   Input Parameters:
3692 + comm - The communicator for the `DM` object
3693 . dim  - The dimension
3694 - R    - The radius
3695 
3696   Output Parameter:
3697 . dm - The `DM` object
3698 
3699   Options Database Key:
3700 . bd_dm_refine - This will refine the surface mesh preserving the sphere geometry
3701 
3702   Level: beginner
3703 
3704 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3705 @*/
3706 PetscErrorCode DMPlexCreateBallMesh(MPI_Comm comm, PetscInt dim, PetscReal R, DM *dm)
3707 {
3708   PetscFunctionBegin;
3709   PetscCall(DMCreate(comm, dm));
3710   PetscCall(DMSetType(*dm, DMPLEX));
3711   PetscCall(DMPlexCreateBallMesh_Internal(*dm, dim, R));
3712   PetscFunctionReturn(PETSC_SUCCESS);
3713 }
3714 
3715 static PetscErrorCode DMPlexCreateReferenceCell_Internal(DM rdm, DMPolytopeType ct)
3716 {
3717   PetscFunctionBegin;
3718   switch (ct) {
3719   case DM_POLYTOPE_POINT: {
3720     PetscInt    numPoints[1]        = {1};
3721     PetscInt    coneSize[1]         = {0};
3722     PetscInt    cones[1]            = {0};
3723     PetscInt    coneOrientations[1] = {0};
3724     PetscScalar vertexCoords[1]     = {0.0};
3725 
3726     PetscCall(DMSetDimension(rdm, 0));
3727     PetscCall(DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3728   } break;
3729   case DM_POLYTOPE_SEGMENT: {
3730     PetscInt    numPoints[2]        = {2, 1};
3731     PetscInt    coneSize[3]         = {2, 0, 0};
3732     PetscInt    cones[2]            = {1, 2};
3733     PetscInt    coneOrientations[2] = {0, 0};
3734     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3735 
3736     PetscCall(DMSetDimension(rdm, 1));
3737     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3738   } break;
3739   case DM_POLYTOPE_POINT_PRISM_TENSOR: {
3740     PetscInt    numPoints[2]        = {2, 1};
3741     PetscInt    coneSize[3]         = {2, 0, 0};
3742     PetscInt    cones[2]            = {1, 2};
3743     PetscInt    coneOrientations[2] = {0, 0};
3744     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3745 
3746     PetscCall(DMSetDimension(rdm, 1));
3747     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3748   } break;
3749   case DM_POLYTOPE_TRIANGLE: {
3750     PetscInt    numPoints[2]        = {3, 1};
3751     PetscInt    coneSize[4]         = {3, 0, 0, 0};
3752     PetscInt    cones[3]            = {1, 2, 3};
3753     PetscInt    coneOrientations[3] = {0, 0, 0};
3754     PetscScalar vertexCoords[6]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0};
3755 
3756     PetscCall(DMSetDimension(rdm, 2));
3757     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3758   } break;
3759   case DM_POLYTOPE_QUADRILATERAL: {
3760     PetscInt    numPoints[2]        = {4, 1};
3761     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3762     PetscInt    cones[4]            = {1, 2, 3, 4};
3763     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3764     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0};
3765 
3766     PetscCall(DMSetDimension(rdm, 2));
3767     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3768   } break;
3769   case DM_POLYTOPE_SEG_PRISM_TENSOR: {
3770     PetscInt    numPoints[2]        = {4, 1};
3771     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3772     PetscInt    cones[4]            = {1, 2, 3, 4};
3773     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3774     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0};
3775 
3776     PetscCall(DMSetDimension(rdm, 2));
3777     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3778   } break;
3779   case DM_POLYTOPE_TETRAHEDRON: {
3780     PetscInt    numPoints[2]        = {4, 1};
3781     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3782     PetscInt    cones[4]            = {1, 2, 3, 4};
3783     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3784     PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0};
3785 
3786     PetscCall(DMSetDimension(rdm, 3));
3787     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3788   } break;
3789   case DM_POLYTOPE_HEXAHEDRON: {
3790     PetscInt    numPoints[2]        = {8, 1};
3791     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3792     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3793     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3794     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3795 
3796     PetscCall(DMSetDimension(rdm, 3));
3797     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3798   } break;
3799   case DM_POLYTOPE_TRI_PRISM: {
3800     PetscInt    numPoints[2]        = {6, 1};
3801     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3802     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3803     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3804     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3805 
3806     PetscCall(DMSetDimension(rdm, 3));
3807     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3808   } break;
3809   case DM_POLYTOPE_TRI_PRISM_TENSOR: {
3810     PetscInt    numPoints[2]        = {6, 1};
3811     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3812     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3813     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3814     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3815 
3816     PetscCall(DMSetDimension(rdm, 3));
3817     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3818   } break;
3819   case DM_POLYTOPE_QUAD_PRISM_TENSOR: {
3820     PetscInt    numPoints[2]        = {8, 1};
3821     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3822     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3823     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3824     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3825 
3826     PetscCall(DMSetDimension(rdm, 3));
3827     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3828   } break;
3829   case DM_POLYTOPE_PYRAMID: {
3830     PetscInt    numPoints[2]        = {5, 1};
3831     PetscInt    coneSize[6]         = {5, 0, 0, 0, 0, 0};
3832     PetscInt    cones[5]            = {1, 2, 3, 4, 5};
3833     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3834     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, 0.0, 0.0, 1.0};
3835 
3836     PetscCall(DMSetDimension(rdm, 3));
3837     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3838   } break;
3839   default:
3840     SETERRQ(PetscObjectComm((PetscObject)rdm), PETSC_ERR_ARG_WRONG, "Cannot create reference cell for cell type %s", DMPolytopeTypes[ct]);
3841   }
3842   {
3843     PetscInt Nv, v;
3844 
3845     /* Must create the celltype label here so that we do not automatically try to compute the types */
3846     PetscCall(DMCreateLabel(rdm, "celltype"));
3847     PetscCall(DMPlexSetCellType(rdm, 0, ct));
3848     PetscCall(DMPlexGetChart(rdm, NULL, &Nv));
3849     for (v = 1; v < Nv; ++v) PetscCall(DMPlexSetCellType(rdm, v, DM_POLYTOPE_POINT));
3850   }
3851   PetscCall(DMPlexInterpolateInPlace_Internal(rdm));
3852   PetscCall(PetscObjectSetName((PetscObject)rdm, DMPolytopeTypes[ct]));
3853   PetscFunctionReturn(PETSC_SUCCESS);
3854 }
3855 
3856 /*@
3857   DMPlexCreateReferenceCell - Create a `DMPLEX` with the appropriate FEM reference cell
3858 
3859   Collective
3860 
3861   Input Parameters:
3862 + comm - The communicator
3863 - ct   - The cell type of the reference cell
3864 
3865   Output Parameter:
3866 . refdm - The reference cell
3867 
3868   Level: intermediate
3869 
3870 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBoxMesh()`
3871 @*/
3872 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, DMPolytopeType ct, DM *refdm)
3873 {
3874   PetscFunctionBegin;
3875   PetscCall(DMCreate(comm, refdm));
3876   PetscCall(DMSetType(*refdm, DMPLEX));
3877   PetscCall(DMPlexCreateReferenceCell_Internal(*refdm, ct));
3878   PetscFunctionReturn(PETSC_SUCCESS);
3879 }
3880 
3881 static PetscErrorCode DMPlexCreateBoundaryLabel_Private(DM dm, const char name[])
3882 {
3883   DM        plex;
3884   DMLabel   label;
3885   PetscBool hasLabel;
3886 
3887   PetscFunctionBegin;
3888   PetscCall(DMHasLabel(dm, name, &hasLabel));
3889   if (hasLabel) PetscFunctionReturn(PETSC_SUCCESS);
3890   PetscCall(DMCreateLabel(dm, name));
3891   PetscCall(DMGetLabel(dm, name, &label));
3892   PetscCall(DMConvert(dm, DMPLEX, &plex));
3893   PetscCall(DMPlexMarkBoundaryFaces(plex, 1, label));
3894   PetscCall(DMPlexLabelComplete(plex, label));
3895   PetscCall(DMDestroy(&plex));
3896   PetscFunctionReturn(PETSC_SUCCESS);
3897 }
3898 
3899 /*
3900   We use the last coordinate as the radius, the inner radius is lower[dim-1] and the outer radius is upper[dim-1]. Then we map the first coordinate around the circle.
3901 
3902     (x, y) -> (r, theta) = (x[1], (x[0] - lower[0]) * 2\pi/(upper[0] - lower[0]))
3903 */
3904 static void boxToAnnulus(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
3905 {
3906   const PetscReal low = PetscRealPart(constants[0]);
3907   const PetscReal upp = PetscRealPart(constants[1]);
3908   const PetscReal r   = PetscRealPart(u[1]);
3909   const PetscReal th  = 2. * PETSC_PI * (PetscRealPart(u[0]) - low) / (upp - low);
3910 
3911   f0[0] = r * PetscCosReal(th);
3912   f0[1] = r * PetscSinReal(th);
3913 }
3914 
3915 PETSC_EXTERN PetscErrorCode PetscOptionsFindPairPrefix_Private(PetscOptions, const char pre[], const char name[], const char *option[], const char *value[], PetscBool *flg);
3916 
3917 const char *const DMPlexShapes[] = {"box", "box_surface", "ball", "sphere", "cylinder", "schwarz_p", "gyroid", "doublet", "annulus", "hypercubic", "zbox", "unknown", "DMPlexShape", "DM_SHAPE_", NULL};
3918 
3919 static PetscErrorCode DMPlexCreateFromOptions_Internal(PetscOptionItems *PetscOptionsObject, PetscBool *useCoordSpace, DM dm)
3920 {
3921   DMPlexShape    shape   = DM_SHAPE_BOX;
3922   DMPolytopeType cell    = DM_POLYTOPE_TRIANGLE;
3923   PetscInt       dim     = 2;
3924   PetscBool      simplex = PETSC_TRUE, interpolate = PETSC_TRUE, adjCone = PETSC_FALSE, adjClosure = PETSC_TRUE, refDomain = PETSC_FALSE;
3925   PetscBool      flg, flg2, fflg, strflg, bdfflg, nameflg;
3926   MPI_Comm       comm;
3927   char           filename[PETSC_MAX_PATH_LEN]   = "<unspecified>";
3928   char           bdFilename[PETSC_MAX_PATH_LEN] = "<unspecified>";
3929   char           plexname[PETSC_MAX_PATH_LEN]   = "";
3930   const char    *option;
3931 
3932   PetscFunctionBegin;
3933   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromOptions, dm, 0, 0, 0));
3934   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3935   /* TODO Turn this into a registration interface */
3936   PetscCall(PetscOptionsString("-dm_plex_filename", "File containing a mesh", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &fflg));
3937   PetscCall(PetscOptionsString("-dm_plex_file_contents", "Contents of a file format in a string", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &strflg));
3938   PetscCall(PetscOptionsString("-dm_plex_boundary_filename", "File containing a mesh boundary", "DMPlexCreateFromFile", bdFilename, bdFilename, sizeof(bdFilename), &bdfflg));
3939   PetscCall(PetscOptionsString("-dm_plex_name", "Name of the mesh in the file", "DMPlexCreateFromFile", plexname, plexname, sizeof(plexname), &nameflg));
3940   PetscCall(PetscOptionsEnum("-dm_plex_cell", "Cell shape", "", DMPolytopeTypes, (PetscEnum)cell, (PetscEnum *)&cell, NULL));
3941   PetscCall(PetscOptionsBool("-dm_plex_reference_cell_domain", "Use a reference cell domain", "", refDomain, &refDomain, NULL));
3942   PetscCall(PetscOptionsEnum("-dm_plex_shape", "Shape for built-in mesh", "", DMPlexShapes, (PetscEnum)shape, (PetscEnum *)&shape, &flg));
3943   PetscCall(PetscOptionsBoundedInt("-dm_plex_dim", "Topological dimension of the mesh", "DMGetDimension", dim, &dim, &flg, 0));
3944   PetscCall(PetscOptionsBool("-dm_plex_simplex", "Mesh cell shape", "", simplex, &simplex, &flg));
3945   PetscCall(PetscOptionsBool("-dm_plex_interpolate", "Flag to create edges and faces automatically", "", interpolate, &interpolate, &flg));
3946   PetscCall(PetscOptionsBool("-dm_plex_adj_cone", "Set adjacency direction", "DMSetBasicAdjacency", adjCone, &adjCone, &flg));
3947   PetscCall(PetscOptionsBool("-dm_plex_adj_closure", "Set adjacency size", "DMSetBasicAdjacency", adjClosure, &adjClosure, &flg2));
3948   if (flg || flg2) PetscCall(DMSetBasicAdjacency(dm, adjCone, adjClosure));
3949 
3950   switch (cell) {
3951   case DM_POLYTOPE_POINT:
3952   case DM_POLYTOPE_SEGMENT:
3953   case DM_POLYTOPE_POINT_PRISM_TENSOR:
3954   case DM_POLYTOPE_TRIANGLE:
3955   case DM_POLYTOPE_QUADRILATERAL:
3956   case DM_POLYTOPE_TETRAHEDRON:
3957   case DM_POLYTOPE_HEXAHEDRON:
3958     *useCoordSpace = PETSC_TRUE;
3959     break;
3960   default:
3961     *useCoordSpace = PETSC_FALSE;
3962     break;
3963   }
3964 
3965   if (fflg) {
3966     DM          dmnew;
3967     const char *name;
3968 
3969     PetscCall(PetscObjectGetName((PetscObject)dm, &name));
3970     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), filename, nameflg ? plexname : name, interpolate, &dmnew));
3971     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3972     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3973   } else if (refDomain) {
3974     PetscCall(DMPlexCreateReferenceCell_Internal(dm, cell));
3975   } else if (bdfflg) {
3976     DM          bdm, dmnew;
3977     const char *name;
3978 
3979     PetscCall(PetscObjectGetName((PetscObject)dm, &name));
3980     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), bdFilename, nameflg ? plexname : name, interpolate, &bdm));
3981     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)bdm, "bd_"));
3982     PetscCall(DMSetFromOptions(bdm));
3983     PetscCall(DMPlexGenerate(bdm, NULL, interpolate, &dmnew));
3984     PetscCall(DMDestroy(&bdm));
3985     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3986     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3987   } else if (strflg) {
3988     DM          dmnew;
3989     PetscViewer viewer;
3990     const char *contents;
3991     char       *strname;
3992     char        tmpdir[PETSC_MAX_PATH_LEN];
3993     char        tmpfilename[PETSC_MAX_PATH_LEN];
3994     char        name[PETSC_MAX_PATH_LEN];
3995     MPI_Comm    comm;
3996     PetscMPIInt rank;
3997 
3998     PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3999     PetscCallMPI(MPI_Comm_rank(comm, &rank));
4000     PetscCall(PetscStrchr(filename, ':', &strname));
4001     PetscCheck(strname, comm, PETSC_ERR_ARG_WRONG, "File contents must have the form \"ext:string_name\", not %s", filename);
4002     strname[0] = '\0';
4003     ++strname;
4004     PetscCall(PetscDLSym(NULL, strname, (void **)&contents));
4005     PetscCheck(contents, comm, PETSC_ERR_ARG_WRONG, "Could not locate mesh string %s", strname);
4006     PetscCall(PetscGetTmp(comm, tmpdir, PETSC_MAX_PATH_LEN));
4007     PetscCall(PetscStrlcat(tmpdir, "/meshXXXXXX", PETSC_MAX_PATH_LEN));
4008     PetscCall(PetscMkdtemp(tmpdir));
4009     PetscCall(PetscSNPrintf(tmpfilename, PETSC_MAX_PATH_LEN, "%s/mesh.%s", tmpdir, filename));
4010     PetscCall(PetscViewerASCIIOpen(comm, tmpfilename, &viewer));
4011     PetscCall(PetscViewerASCIIPrintf(viewer, "%s\n", contents));
4012     PetscCall(PetscViewerDestroy(&viewer));
4013     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), tmpfilename, plexname, interpolate, &dmnew));
4014     PetscCall(PetscRMTree(tmpdir));
4015     PetscCall(PetscSNPrintf(name, PETSC_MAX_PATH_LEN, "%s Mesh", strname));
4016     PetscCall(PetscObjectSetName((PetscObject)dm, name));
4017     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4018     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4019   } else {
4020     PetscCall(PetscObjectSetName((PetscObject)dm, DMPlexShapes[shape]));
4021     switch (shape) {
4022     case DM_SHAPE_BOX:
4023     case DM_SHAPE_ZBOX:
4024     case DM_SHAPE_ANNULUS: {
4025       PetscInt       faces[3]  = {0, 0, 0};
4026       PetscReal      lower[3]  = {0, 0, 0};
4027       PetscReal      upper[3]  = {1, 1, 1};
4028       DMBoundaryType bdt[3]    = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
4029       PetscBool      isAnnular = shape == DM_SHAPE_ANNULUS ? PETSC_TRUE : PETSC_FALSE;
4030       PetscInt       i, n;
4031 
4032       n = dim;
4033       for (i = 0; i < dim; ++i) faces[i] = (dim == 1 ? 1 : 4 - dim);
4034       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
4035       n = 3;
4036       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
4037       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4038       n = 3;
4039       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
4040       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4041       n = 3;
4042       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
4043       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4044 
4045       PetscCheck(!isAnnular || dim == 2, comm, PETSC_ERR_ARG_OUTOFRANGE, "Only two dimensional annuli have been implemented");
4046       if (isAnnular)
4047         for (i = 0; i < dim - 1; ++i) bdt[i] = DM_BOUNDARY_PERIODIC;
4048 
4049       switch (cell) {
4050       case DM_POLYTOPE_TRI_PRISM_TENSOR:
4051         PetscCall(DMPlexCreateWedgeBoxMesh_Internal(dm, faces, lower, upper, bdt));
4052         if (!interpolate) {
4053           DM udm;
4054 
4055           PetscCall(DMPlexUninterpolate(dm, &udm));
4056           PetscCall(DMPlexReplace_Internal(dm, &udm));
4057         }
4058         break;
4059       default:
4060         PetscCall(DMPlexCreateBoxMesh_Internal(dm, shape, dim, simplex, faces, lower, upper, bdt, interpolate));
4061         break;
4062       }
4063       if (isAnnular) {
4064         DM          cdm;
4065         PetscDS     cds;
4066         PetscScalar bounds[2] = {lower[0], upper[0]};
4067 
4068         // Fix coordinates for annular region
4069         PetscCall(DMSetPeriodicity(dm, NULL, NULL, NULL));
4070         PetscCall(DMSetCellCoordinatesLocal(dm, NULL));
4071         PetscCall(DMSetCellCoordinates(dm, NULL));
4072         PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_TRUE, NULL));
4073         PetscCall(DMGetCoordinateDM(dm, &cdm));
4074         PetscCall(DMGetDS(cdm, &cds));
4075         PetscCall(PetscDSSetConstants(cds, 2, bounds));
4076         PetscCall(DMPlexRemapGeometry(dm, 0.0, boxToAnnulus));
4077       }
4078     } break;
4079     case DM_SHAPE_BOX_SURFACE: {
4080       PetscInt  faces[3] = {0, 0, 0};
4081       PetscReal lower[3] = {0, 0, 0};
4082       PetscReal upper[3] = {1, 1, 1};
4083       PetscInt  i, n;
4084 
4085       n = dim + 1;
4086       for (i = 0; i < dim + 1; ++i) faces[i] = (dim + 1 == 1 ? 1 : 4 - (dim + 1));
4087       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
4088       n = 3;
4089       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
4090       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
4091       n = 3;
4092       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
4093       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
4094       PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(dm, dim + 1, faces, lower, upper, interpolate));
4095     } break;
4096     case DM_SHAPE_SPHERE: {
4097       PetscReal R = 1.0;
4098 
4099       PetscCall(PetscOptionsReal("-dm_plex_sphere_radius", "Radius of the sphere", "", R, &R, &flg));
4100       PetscCall(DMPlexCreateSphereMesh_Internal(dm, dim, simplex, R));
4101     } break;
4102     case DM_SHAPE_BALL: {
4103       PetscReal R = 1.0;
4104 
4105       PetscCall(PetscOptionsReal("-dm_plex_ball_radius", "Radius of the ball", "", R, &R, &flg));
4106       PetscCall(DMPlexCreateBallMesh_Internal(dm, dim, R));
4107     } break;
4108     case DM_SHAPE_CYLINDER: {
4109       DMBoundaryType bdt = DM_BOUNDARY_NONE;
4110       PetscInt       Nw  = 6;
4111       PetscInt       Nr  = 0;
4112 
4113       PetscCall(PetscOptionsEnum("-dm_plex_cylinder_bd", "Boundary type in the z direction", "", DMBoundaryTypes, (PetscEnum)bdt, (PetscEnum *)&bdt, NULL));
4114       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_wedges", "Number of wedges around the cylinder", "", Nw, &Nw, NULL));
4115       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_refine", "Number of refinements before projection", "", Nr, &Nr, NULL));
4116       switch (cell) {
4117       case DM_POLYTOPE_TRI_PRISM_TENSOR:
4118         PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(dm, Nw, interpolate));
4119         break;
4120       default:
4121         PetscCall(DMPlexCreateHexCylinderMesh_Internal(dm, bdt, Nr));
4122         break;
4123       }
4124     } break;
4125     case DM_SHAPE_SCHWARZ_P: // fallthrough
4126     case DM_SHAPE_GYROID: {
4127       PetscInt       extent[3] = {1, 1, 1}, refine = 0, layers = 0, three;
4128       PetscReal      thickness   = 0.;
4129       DMBoundaryType periodic[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
4130       DMPlexTPSType  tps_type    = shape == DM_SHAPE_SCHWARZ_P ? DMPLEX_TPS_SCHWARZ_P : DMPLEX_TPS_GYROID;
4131       PetscBool      tps_distribute;
4132       PetscCall(PetscOptionsIntArray("-dm_plex_tps_extent", "Number of replicas for each of three dimensions", NULL, extent, (three = 3, &three), NULL));
4133       PetscCall(PetscOptionsInt("-dm_plex_tps_refine", "Number of refinements", NULL, refine, &refine, NULL));
4134       PetscCall(PetscOptionsEnumArray("-dm_plex_tps_periodic", "Periodicity in each of three dimensions", NULL, DMBoundaryTypes, (PetscEnum *)periodic, (three = 3, &three), NULL));
4135       PetscCall(PetscOptionsInt("-dm_plex_tps_layers", "Number of layers in volumetric extrusion (or zero to not extrude)", NULL, layers, &layers, NULL));
4136       PetscCall(PetscOptionsReal("-dm_plex_tps_thickness", "Thickness of volumetric extrusion", NULL, thickness, &thickness, NULL));
4137       PetscCall(DMPlexDistributeGetDefault(dm, &tps_distribute));
4138       PetscCall(PetscOptionsBool("-dm_plex_tps_distribute", "Distribute the 2D mesh prior to refinement and extrusion", NULL, tps_distribute, &tps_distribute, NULL));
4139       PetscCall(DMPlexCreateTPSMesh_Internal(dm, tps_type, extent, periodic, tps_distribute, refine, layers, thickness));
4140     } break;
4141     case DM_SHAPE_DOUBLET: {
4142       DM        dmnew;
4143       PetscReal rl = 0.0;
4144 
4145       PetscCall(PetscOptionsReal("-dm_plex_doublet_refinementlimit", "Refinement limit", NULL, rl, &rl, NULL));
4146       PetscCall(DMPlexCreateDoublet(PetscObjectComm((PetscObject)dm), dim, simplex, interpolate, rl, &dmnew));
4147       PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4148       PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4149     } break;
4150     case DM_SHAPE_HYPERCUBIC: {
4151       PetscInt       *edges;
4152       PetscReal      *lower, *upper;
4153       DMBoundaryType *bdt;
4154       PetscInt        n, d;
4155 
4156       *useCoordSpace = PETSC_FALSE;
4157       PetscCall(PetscMalloc4(dim, &edges, dim, &lower, dim, &upper, dim, &bdt));
4158       for (d = 0; d < dim; ++d) {
4159         edges[d] = 1;
4160         lower[d] = 0.;
4161         upper[d] = 1.;
4162         bdt[d]   = DM_BOUNDARY_PERIODIC;
4163       }
4164       n = dim;
4165       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", edges, &n, &flg));
4166       n = dim;
4167       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
4168       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4169       n = dim;
4170       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
4171       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4172       n = dim;
4173       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
4174       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4175       PetscCall(DMPlexCreateHypercubicMesh_Internal(dm, dim, lower, upper, edges, bdt));
4176       PetscCall(PetscFree4(edges, lower, upper, bdt));
4177     } break;
4178     default:
4179       SETERRQ(comm, PETSC_ERR_SUP, "Domain shape %s is unsupported", DMPlexShapes[shape]);
4180     }
4181   }
4182   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
4183   if (!((PetscObject)dm)->name && nameflg) PetscCall(PetscObjectSetName((PetscObject)dm, plexname));
4184   // Allow label creation
4185   PetscCall(PetscOptionsFindPairPrefix_Private(NULL, ((PetscObject)dm)->prefix, "-dm_plex_label_", &option, NULL, &flg));
4186   if (flg) {
4187     DMLabel     label;
4188     PetscInt    points[1024], n = 1024;
4189     char        fulloption[PETSC_MAX_PATH_LEN];
4190     const char *name = &option[14];
4191 
4192     PetscCall(DMCreateLabel(dm, name));
4193     PetscCall(DMGetLabel(dm, name, &label));
4194     fulloption[0] = '-';
4195     fulloption[1] = 0;
4196     PetscCall(PetscStrlcat(fulloption, option, PETSC_MAX_PATH_LEN));
4197     PetscCall(PetscOptionsGetIntArray(NULL, ((PetscObject)dm)->prefix, fulloption, points, &n, NULL));
4198     for (PetscInt p = 0; p < n; ++p) PetscCall(DMLabelSetValue(label, points[p], 1));
4199   }
4200   // Allow cohesive label creation
4201   //   Faces are input, completed, and all points are marked with their depth
4202   PetscCall(PetscOptionsFindPairPrefix_Private(NULL, ((PetscObject)dm)->prefix, "-dm_plex_cohesive_label_", &option, NULL, &flg));
4203   if (flg) {
4204     DMLabel  label;
4205     PetscInt points[1024], n, pStart, pEnd, Nl = 1;
4206     char     fulloption[PETSC_MAX_PATH_LEN];
4207     char     name[PETSC_MAX_PATH_LEN];
4208     size_t   len;
4209 
4210     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
4211     PetscCall(PetscStrncpy(name, &option[23], PETSC_MAX_PATH_LEN));
4212     PetscCall(PetscStrlen(name, &len));
4213     if (name[len - 1] == '0') Nl = 10;
4214     for (PetscInt l = 0; l < Nl; ++l) {
4215       if (l > 0) name[len - 1] = '0' + l;
4216       fulloption[0] = 0;
4217       PetscCall(PetscStrlcat(fulloption, "-dm_plex_cohesive_label_", 32));
4218       PetscCall(PetscStrlcat(fulloption, name, PETSC_MAX_PATH_LEN - 32));
4219       n = 1024;
4220       PetscCall(PetscOptionsGetIntArray(NULL, ((PetscObject)dm)->prefix, fulloption, points, &n, &flg));
4221       if (!flg) break;
4222       PetscCall(DMCreateLabel(dm, name));
4223       PetscCall(DMGetLabel(dm, name, &label));
4224       if (pStart >= pEnd) n = 0;
4225       for (PetscInt p = 0; p < n; ++p) {
4226         const PetscInt point   = points[p];
4227         PetscInt      *closure = NULL;
4228         PetscInt       clSize, pdepth;
4229 
4230         PetscCall(DMPlexGetPointDepth(dm, point, &pdepth));
4231         PetscCall(DMLabelSetValue(label, point, pdepth));
4232         PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &clSize, &closure));
4233         for (PetscInt cl = 0; cl < clSize * 2; cl += 2) {
4234           PetscCall(DMPlexGetPointDepth(dm, closure[cl], &pdepth));
4235           PetscCall(DMLabelSetValue(label, closure[cl], pdepth));
4236         }
4237         PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &clSize, &closure));
4238       }
4239       PetscCall(DMPlexOrientLabel(dm, label));
4240       PetscCall(DMPlexLabelCohesiveComplete(dm, label, NULL, 1, PETSC_FALSE, PETSC_FALSE, NULL));
4241     }
4242   }
4243   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromOptions, dm, 0, 0, 0));
4244   PetscFunctionReturn(PETSC_SUCCESS);
4245 }
4246 
4247 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
4248 {
4249   DM_Plex  *mesh = (DM_Plex *)dm->data;
4250   PetscBool flg, flg2;
4251   char      bdLabel[PETSC_MAX_PATH_LEN];
4252   char      method[PETSC_MAX_PATH_LEN];
4253 
4254   PetscFunctionBegin;
4255   /* Handle viewing */
4256   PetscCall(PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL));
4257   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fem", "Debug output level for all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL, 0));
4258   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fvm", "Debug output level for all fvm computations", "DMPlexSNESComputeResidualFVM", 0, &mesh->printFVM, NULL, 0));
4259   PetscCall(PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL));
4260   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL, 0));
4261   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_locate", "Debug output level all point location computations", "DMLocatePoints", 0, &mesh->printLocate, NULL, 0));
4262   PetscCall(DMMonitorSetFromOptions(dm, "-dm_plex_monitor_throughput", "Monitor the simulation throughput", "DMPlexMonitorThroughput", DMPlexMonitorThroughput, NULL, &flg));
4263   if (flg) PetscCall(PetscLogDefaultBegin());
4264   /* Labeling */
4265   PetscCall(PetscOptionsString("-dm_plex_boundary_label", "Label to mark the mesh boundary", "", bdLabel, bdLabel, sizeof(bdLabel), &flg));
4266   if (flg) PetscCall(DMPlexCreateBoundaryLabel_Private(dm, bdLabel));
4267   /* Point Location */
4268   PetscCall(PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL));
4269   /* Partitioning and distribution */
4270   PetscCall(PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL));
4271   /* Reordering */
4272   PetscCall(PetscOptionsBool("-dm_reorder_section", "Compute point permutation for local section", "DMReorderSectionSetDefault", PETSC_FALSE, &flg2, &flg));
4273   if (flg) PetscCall(DMReorderSectionSetDefault(dm, flg2 ? DM_REORDER_DEFAULT_TRUE : DM_REORDER_DEFAULT_FALSE));
4274   PetscCall(PetscOptionsString("-dm_reorder_section_type", "Reordering method for local section", "DMReorderSectionSetType", method, method, PETSC_MAX_PATH_LEN, &flg));
4275   if (flg) PetscCall(DMReorderSectionSetType(dm, method));
4276   /* Generation and remeshing */
4277   PetscCall(PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL));
4278   /* Projection behavior */
4279   PetscCall(PetscOptionsBoundedInt("-dm_plex_max_projection_height", "Maximum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL, 0));
4280   PetscCall(PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL));
4281   /* Checking structure */
4282   {
4283     PetscBool all = PETSC_FALSE;
4284 
4285     PetscCall(PetscOptionsBool("-dm_plex_check_all", "Perform all basic checks", "DMPlexCheck", PETSC_FALSE, &all, NULL));
4286     if (all) {
4287       PetscCall(DMPlexCheck(dm));
4288     } else {
4289       PetscCall(PetscOptionsBool("-dm_plex_check_symmetry", "Check that the adjacency information in the mesh is symmetric", "DMPlexCheckSymmetry", PETSC_FALSE, &flg, &flg2));
4290       if (flg && flg2) PetscCall(DMPlexCheckSymmetry(dm));
4291       PetscCall(PetscOptionsBool("-dm_plex_check_skeleton", "Check that each cell has the correct number of vertices (only for homogeneous simplex or tensor meshes)", "DMPlexCheckSkeleton", PETSC_FALSE, &flg, &flg2));
4292       if (flg && flg2) PetscCall(DMPlexCheckSkeleton(dm, 0));
4293       PetscCall(PetscOptionsBool("-dm_plex_check_faces", "Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type", "DMPlexCheckFaces", PETSC_FALSE, &flg, &flg2));
4294       if (flg && flg2) PetscCall(DMPlexCheckFaces(dm, 0));
4295       PetscCall(PetscOptionsBool("-dm_plex_check_geometry", "Check that cells have positive volume", "DMPlexCheckGeometry", PETSC_FALSE, &flg, &flg2));
4296       if (flg && flg2) PetscCall(DMPlexCheckGeometry(dm));
4297       PetscCall(PetscOptionsBool("-dm_plex_check_pointsf", "Check some necessary conditions for PointSF", "DMPlexCheckPointSF", PETSC_FALSE, &flg, &flg2));
4298       if (flg && flg2) PetscCall(DMPlexCheckPointSF(dm, NULL, PETSC_FALSE));
4299       PetscCall(PetscOptionsBool("-dm_plex_check_interface_cones", "Check points on inter-partition interfaces have conforming order of cone points", "DMPlexCheckInterfaceCones", PETSC_FALSE, &flg, &flg2));
4300       if (flg && flg2) PetscCall(DMPlexCheckInterfaceCones(dm));
4301     }
4302     PetscCall(PetscOptionsBool("-dm_plex_check_cell_shape", "Check cell shape", "DMPlexCheckCellShape", PETSC_FALSE, &flg, &flg2));
4303     if (flg && flg2) PetscCall(DMPlexCheckCellShape(dm, PETSC_TRUE, PETSC_DETERMINE));
4304   }
4305   {
4306     PetscReal scale = 1.0;
4307 
4308     PetscCall(PetscOptionsReal("-dm_plex_scale", "Scale factor for mesh coordinates", "DMPlexScale", scale, &scale, &flg));
4309     if (flg) {
4310       Vec coordinates, coordinatesLocal;
4311 
4312       PetscCall(DMGetCoordinates(dm, &coordinates));
4313       PetscCall(DMGetCoordinatesLocal(dm, &coordinatesLocal));
4314       PetscCall(VecScale(coordinates, scale));
4315       PetscCall(VecScale(coordinatesLocal, scale));
4316     }
4317   }
4318   PetscCall(PetscPartitionerSetFromOptions(mesh->partitioner));
4319   PetscFunctionReturn(PETSC_SUCCESS);
4320 }
4321 
4322 PetscErrorCode DMSetFromOptions_Overlap_Plex(DM dm, PetscOptionItems *PetscOptionsObject, PetscInt *overlap)
4323 {
4324   PetscInt  numOvLabels = 16, numOvExLabels = 16;
4325   char     *ovLabelNames[16], *ovExLabelNames[16];
4326   PetscInt  numOvValues = 16, numOvExValues = 16, l;
4327   PetscBool flg;
4328 
4329   PetscFunctionBegin;
4330   PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", *overlap, overlap, NULL, 0));
4331   PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_labels", "List of overlap label names", "DMPlexDistribute", ovLabelNames, &numOvLabels, &flg));
4332   if (!flg) numOvLabels = 0;
4333   if (numOvLabels) {
4334     ((DM_Plex *)dm->data)->numOvLabels = numOvLabels;
4335     for (l = 0; l < numOvLabels; ++l) {
4336       PetscCall(DMGetLabel(dm, ovLabelNames[l], &((DM_Plex *)dm->data)->ovLabels[l]));
4337       PetscCheck(((DM_Plex *)dm->data)->ovLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovLabelNames[l]);
4338       PetscCall(PetscFree(ovLabelNames[l]));
4339     }
4340     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_values", "List of overlap label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovValues, &numOvValues, &flg));
4341     if (!flg) numOvValues = 0;
4342     PetscCheck(numOvLabels == numOvValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvLabels, numOvValues);
4343 
4344     PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_exclude_labels", "List of overlap exclude label names", "DMPlexDistribute", ovExLabelNames, &numOvExLabels, &flg));
4345     if (!flg) numOvExLabels = 0;
4346     ((DM_Plex *)dm->data)->numOvExLabels = numOvExLabels;
4347     for (l = 0; l < numOvExLabels; ++l) {
4348       PetscCall(DMGetLabel(dm, ovExLabelNames[l], &((DM_Plex *)dm->data)->ovExLabels[l]));
4349       PetscCheck(((DM_Plex *)dm->data)->ovExLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovExLabelNames[l]);
4350       PetscCall(PetscFree(ovExLabelNames[l]));
4351     }
4352     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_exclude_values", "List of overlap exclude label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovExValues, &numOvExValues, &flg));
4353     if (!flg) numOvExValues = 0;
4354     PetscCheck(numOvExLabels == numOvExValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of exclude labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvExLabels, numOvExValues);
4355   }
4356   PetscFunctionReturn(PETSC_SUCCESS);
4357 }
4358 
4359 static PetscErrorCode DMSetFromOptions_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
4360 {
4361   PetscFunctionList    ordlist;
4362   char                 oname[256];
4363   char                 sublabelname[PETSC_MAX_PATH_LEN] = "";
4364   DMReorderDefaultFlag reorder;
4365   PetscReal            volume    = -1.0;
4366   PetscInt             prerefine = 0, refine = 0, r, coarsen = 0, overlap = 0, extLayers = 0, dim;
4367   PetscBool            uniformOrig = PETSC_FALSE, created = PETSC_FALSE, uniform = PETSC_TRUE, distribute, saveSF = PETSC_FALSE, interpolate = PETSC_TRUE, coordSpace = PETSC_TRUE, remap = PETSC_TRUE, ghostCells = PETSC_FALSE, isHierarchy, flg;
4368 
4369   PetscFunctionBegin;
4370   PetscOptionsHeadBegin(PetscOptionsObject, "DMPlex Options");
4371   if (dm->cloneOpts) goto non_refine;
4372   /* Handle automatic creation */
4373   PetscCall(DMGetDimension(dm, &dim));
4374   if (dim < 0) {
4375     PetscCall(DMPlexCreateFromOptions_Internal(PetscOptionsObject, &coordSpace, dm));
4376     created = PETSC_TRUE;
4377   }
4378   PetscCall(DMGetDimension(dm, &dim));
4379   /* Handle interpolation before distribution */
4380   PetscCall(PetscOptionsBool("-dm_plex_interpolate_pre", "Flag to interpolate mesh before distribution", "", interpolate, &interpolate, &flg));
4381   if (flg) {
4382     DMPlexInterpolatedFlag interpolated;
4383 
4384     PetscCall(DMPlexIsInterpolated(dm, &interpolated));
4385     if (interpolated == DMPLEX_INTERPOLATED_FULL && !interpolate) {
4386       DM udm;
4387 
4388       PetscCall(DMPlexUninterpolate(dm, &udm));
4389       PetscCall(DMPlexReplace_Internal(dm, &udm));
4390     } else if (interpolated != DMPLEX_INTERPOLATED_FULL && interpolate) {
4391       DM idm;
4392 
4393       PetscCall(DMPlexInterpolate(dm, &idm));
4394       PetscCall(DMPlexReplace_Internal(dm, &idm));
4395     }
4396   }
4397   // Handle submesh selection before distribution
4398   PetscCall(PetscOptionsString("-dm_plex_submesh", "Label to use for submesh selection", "", sublabelname, sublabelname, PETSC_MAX_PATH_LEN, &flg));
4399   if (flg) {
4400     DM              subdm;
4401     DMLabel         label;
4402     IS              valueIS, pointIS;
4403     const PetscInt *values, *points;
4404     PetscBool       markedFaces = PETSC_FALSE;
4405     PetscInt        Nv, value, Np;
4406 
4407     PetscCall(DMGetLabel(dm, sublabelname, &label));
4408     PetscCall(DMLabelGetNumValues(label, &Nv));
4409     PetscCheck(Nv == 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Only a single label value is currently supported for submesh selection, not %" PetscInt_FMT, Nv);
4410     PetscCall(DMLabelGetValueIS(label, &valueIS));
4411     PetscCall(ISGetIndices(valueIS, &values));
4412     value = values[0];
4413     PetscCall(ISRestoreIndices(valueIS, &values));
4414     PetscCall(ISDestroy(&valueIS));
4415     PetscCall(DMLabelGetStratumSize(label, value, &Np));
4416     PetscCall(DMLabelGetStratumIS(label, value, &pointIS));
4417     PetscCall(ISGetIndices(pointIS, &points));
4418     for (PetscInt p = 0; p < Np; ++p) {
4419       PetscInt pdepth;
4420 
4421       PetscCall(DMPlexGetPointDepth(dm, points[p], &pdepth));
4422       if (pdepth) {
4423         markedFaces = PETSC_TRUE;
4424         break;
4425       }
4426     }
4427     PetscCall(ISRestoreIndices(pointIS, &points));
4428     PetscCall(ISDestroy(&pointIS));
4429     PetscCall(DMPlexCreateSubmesh(dm, label, value, markedFaces, &subdm));
4430     PetscCall(DMPlexReplace_Internal(dm, &subdm));
4431     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4432   }
4433   /* Handle DMPlex refinement before distribution */
4434   PetscCall(DMPlexGetRefinementUniform(dm, &uniformOrig));
4435   PetscCall(PetscOptionsBoundedInt("-dm_refine_pre", "The number of refinements before distribution", "DMCreate", prerefine, &prerefine, NULL, 0));
4436   PetscCall(PetscOptionsBool("-dm_refine_remap_pre", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
4437   PetscCall(PetscOptionsBool("-dm_refine_uniform_pre", "Flag for uniform refinement before distribution", "DMCreate", uniform, &uniform, &flg));
4438   if (flg) PetscCall(DMPlexSetRefinementUniform(dm, uniform));
4439   PetscCall(PetscOptionsReal("-dm_refine_volume_limit_pre", "The maximum cell volume after refinement before distribution", "DMCreate", volume, &volume, &flg));
4440   if (flg) {
4441     PetscCall(DMPlexSetRefinementUniform(dm, PETSC_FALSE));
4442     PetscCall(DMPlexSetRefinementLimit(dm, volume));
4443     prerefine = PetscMax(prerefine, 1);
4444   }
4445   if (prerefine) PetscCall(DMLocalizeCoordinates(dm));
4446   for (r = 0; r < prerefine; ++r) {
4447     DM             rdm;
4448     PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4449 
4450     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4451     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
4452     PetscCall(DMPlexReplace_Internal(dm, &rdm));
4453     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4454     if (coordFunc && remap) {
4455       PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4456       ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4457     }
4458   }
4459   PetscCall(DMPlexSetRefinementUniform(dm, uniformOrig));
4460   /* Handle DMPlex extrusion before distribution */
4461   PetscCall(PetscOptionsBoundedInt("-dm_extrude", "The number of layers to extrude", "", extLayers, &extLayers, NULL, 0));
4462   if (extLayers) {
4463     DM edm;
4464 
4465     PetscCall(DMExtrude(dm, extLayers, &edm));
4466     PetscCall(DMPlexReplace_Internal(dm, &edm));
4467     ((DM_Plex *)dm->data)->coordFunc = NULL;
4468     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4469     extLayers = 0;
4470     PetscCall(DMGetDimension(dm, &dim));
4471   }
4472   /* Handle DMPlex reordering before distribution */
4473   PetscCall(DMPlexReorderGetDefault(dm, &reorder));
4474   PetscCall(MatGetOrderingList(&ordlist));
4475   PetscCall(PetscStrncpy(oname, MATORDERINGNATURAL, sizeof(oname)));
4476   PetscCall(PetscOptionsFList("-dm_plex_reorder", "Set mesh reordering type", "DMPlexGetOrdering", ordlist, MATORDERINGNATURAL, oname, sizeof(oname), &flg));
4477   if (reorder == DM_REORDER_DEFAULT_TRUE || flg) {
4478     DM pdm;
4479     IS perm;
4480 
4481     PetscCall(DMPlexGetOrdering(dm, oname, NULL, &perm));
4482     PetscCall(DMPlexPermute(dm, perm, &pdm));
4483     PetscCall(ISDestroy(&perm));
4484     PetscCall(DMPlexReplace_Internal(dm, &pdm));
4485     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4486   }
4487   /* Handle DMPlex distribution */
4488   PetscCall(DMPlexDistributeGetDefault(dm, &distribute));
4489   PetscCall(PetscOptionsBool("-dm_distribute", "Flag to redistribute a mesh among processes", "DMPlexDistribute", distribute, &distribute, NULL));
4490   PetscCall(PetscOptionsBool("-dm_distribute_save_sf", "Flag to save the migration SF", "DMPlexSetMigrationSF", saveSF, &saveSF, NULL));
4491   PetscCall(DMSetFromOptions_Overlap_Plex(dm, PetscOptionsObject, &overlap));
4492   if (distribute) {
4493     DM               pdm = NULL;
4494     PetscPartitioner part;
4495     PetscSF          sfMigration;
4496 
4497     PetscCall(DMPlexGetPartitioner(dm, &part));
4498     PetscCall(PetscPartitionerSetFromOptions(part));
4499     PetscCall(DMPlexDistribute(dm, overlap, &sfMigration, &pdm));
4500     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
4501     if (saveSF) PetscCall(DMPlexSetMigrationSF(dm, sfMigration));
4502     PetscCall(PetscSFDestroy(&sfMigration));
4503   }
4504   /* Must check CEED options before creating function space for coordinates */
4505   {
4506     PetscBool useCeed = PETSC_FALSE, flg;
4507 
4508     PetscCall(PetscOptionsBool("-dm_plex_use_ceed", "Use LibCEED as the FEM backend", "DMPlexSetUseCeed", useCeed, &useCeed, &flg));
4509     if (flg) PetscCall(DMPlexSetUseCeed(dm, useCeed));
4510   }
4511   /* Create coordinate space */
4512   if (created) {
4513     DM_Plex  *mesh   = (DM_Plex *)dm->data;
4514     PetscInt  degree = 1, deg;
4515     PetscInt  height = 0;
4516     DM        cdm;
4517     PetscBool flg, localize = PETSC_TRUE, sparseLocalize = PETSC_TRUE;
4518 
4519     PetscCall(PetscOptionsBool("-dm_coord_space", "Use an FEM space for coordinates", "", coordSpace, &coordSpace, &flg));
4520     PetscCall(PetscOptionsInt("-dm_coord_petscspace_degree", "FEM degree for coordinate space", "", degree, &degree, NULL));
4521     PetscCall(DMGetCoordinateDegree_Internal(dm, &deg));
4522     if (coordSpace && deg <= 1) PetscCall(DMPlexCreateCoordinateSpace(dm, degree, PETSC_TRUE, mesh->coordFunc));
4523     PetscCall(DMGetCoordinateDM(dm, &cdm));
4524     if (flg && !coordSpace) {
4525       PetscDS      cds;
4526       PetscObject  obj;
4527       PetscClassId id;
4528 
4529       PetscCall(DMGetDS(cdm, &cds));
4530       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
4531       PetscCall(PetscObjectGetClassId(obj, &id));
4532       if (id == PETSCFE_CLASSID) {
4533         PetscContainer dummy;
4534 
4535         PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &dummy));
4536         PetscCall(PetscObjectSetName((PetscObject)dummy, "coordinates"));
4537         PetscCall(DMSetField(cdm, 0, NULL, (PetscObject)dummy));
4538         PetscCall(PetscContainerDestroy(&dummy));
4539         PetscCall(DMClearDS(cdm));
4540       }
4541       mesh->coordFunc = NULL;
4542     }
4543     PetscCall(PetscOptionsBool("-dm_localize", "Localize mesh coordinates", "", localize, &localize, NULL));
4544     PetscCall(PetscOptionsBool("-dm_sparse_localize", "Localize only necessary cells", "DMSetSparseLocalize", sparseLocalize, &sparseLocalize, &flg));
4545     if (flg) PetscCall(DMSetSparseLocalize(dm, sparseLocalize));
4546     PetscCall(PetscOptionsInt("-dm_localize_height", "Localize edges and faces in addition to cells", "", height, &height, &flg));
4547     if (flg) PetscCall(DMPlexSetMaxProjectionHeight(cdm, height));
4548     if (localize) PetscCall(DMLocalizeCoordinates(dm));
4549   }
4550   /* Handle DMPlex refinement */
4551   remap = PETSC_TRUE;
4552   PetscCall(PetscOptionsBoundedInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL, 0));
4553   PetscCall(PetscOptionsBool("-dm_refine_remap", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
4554   PetscCall(PetscOptionsBoundedInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy, 0));
4555   if (refine) PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
4556   if (refine && isHierarchy) {
4557     DM *dms, coarseDM;
4558 
4559     PetscCall(DMGetCoarseDM(dm, &coarseDM));
4560     PetscCall(PetscObjectReference((PetscObject)coarseDM));
4561     PetscCall(PetscMalloc1(refine, &dms));
4562     PetscCall(DMRefineHierarchy(dm, refine, dms));
4563     /* Total hack since we do not pass in a pointer */
4564     PetscCall(DMPlexSwap_Static(dm, dms[refine - 1]));
4565     if (refine == 1) {
4566       PetscCall(DMSetCoarseDM(dm, dms[0]));
4567       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4568     } else {
4569       PetscCall(DMSetCoarseDM(dm, dms[refine - 2]));
4570       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4571       PetscCall(DMSetCoarseDM(dms[0], dms[refine - 1]));
4572       PetscCall(DMPlexSetRegularRefinement(dms[0], PETSC_TRUE));
4573     }
4574     PetscCall(DMSetCoarseDM(dms[refine - 1], coarseDM));
4575     PetscCall(PetscObjectDereference((PetscObject)coarseDM));
4576     /* Free DMs */
4577     for (r = 0; r < refine; ++r) {
4578       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4579       PetscCall(DMDestroy(&dms[r]));
4580     }
4581     PetscCall(PetscFree(dms));
4582   } else {
4583     for (r = 0; r < refine; ++r) {
4584       DM             rdm;
4585       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4586 
4587       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4588       PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
4589       /* Total hack since we do not pass in a pointer */
4590       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4591       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4592       if (coordFunc && remap) {
4593         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4594         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4595       }
4596     }
4597   }
4598   /* Handle DMPlex coarsening */
4599   PetscCall(PetscOptionsBoundedInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL, 0));
4600   PetscCall(PetscOptionsBoundedInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy, 0));
4601   if (coarsen && isHierarchy) {
4602     DM *dms;
4603 
4604     PetscCall(PetscMalloc1(coarsen, &dms));
4605     PetscCall(DMCoarsenHierarchy(dm, coarsen, dms));
4606     /* Free DMs */
4607     for (r = 0; r < coarsen; ++r) {
4608       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4609       PetscCall(DMDestroy(&dms[r]));
4610     }
4611     PetscCall(PetscFree(dms));
4612   } else {
4613     for (r = 0; r < coarsen; ++r) {
4614       DM             cdm;
4615       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4616 
4617       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4618       PetscCall(DMCoarsen(dm, PetscObjectComm((PetscObject)dm), &cdm));
4619       /* Total hack since we do not pass in a pointer */
4620       PetscCall(DMPlexReplace_Internal(dm, &cdm));
4621       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4622       if (coordFunc) {
4623         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4624         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4625       }
4626     }
4627   }
4628   // Handle coordinate remapping
4629   remap = PETSC_FALSE;
4630   PetscCall(PetscOptionsBool("-dm_coord_remap", "Flag to control coordinate remapping", "", remap, &remap, NULL));
4631   if (remap) {
4632     DMPlexCoordMap map     = DM_COORD_MAP_NONE;
4633     PetscPointFunc mapFunc = NULL;
4634     PetscScalar    params[16];
4635     PetscInt       Np = PETSC_STATIC_ARRAY_LENGTH(params), cdim;
4636     MPI_Comm       comm;
4637 
4638     PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
4639     PetscCall(DMGetCoordinateDim(dm, &cdim));
4640     PetscCall(PetscOptionsScalarArray("-dm_coord_map_params", "Parameters for the coordinate remapping", "", params, &Np, &flg));
4641     if (!flg) Np = 0;
4642     // TODO Allow user to pass a map function by name
4643     PetscCall(PetscOptionsEnum("-dm_coord_map", "Coordinate mapping for built-in mesh", "", DMPlexCoordMaps, (PetscEnum)map, (PetscEnum *)&map, &flg));
4644     if (flg) {
4645       switch (map) {
4646       case DM_COORD_MAP_NONE:
4647         mapFunc = coordMap_identity;
4648         break;
4649       case DM_COORD_MAP_SHEAR:
4650         mapFunc = coordMap_shear;
4651         if (!Np) {
4652           Np        = cdim + 1;
4653           params[0] = 0;
4654           for (PetscInt d = 1; d <= cdim; ++d) params[d] = 1.0;
4655         }
4656         PetscCheck(Np == cdim + 1, comm, PETSC_ERR_ARG_WRONG, "The shear coordinate map must have cdim + 1 = %" PetscInt_FMT " parameters, not %" PetscInt_FMT, cdim + 1, Np);
4657         break;
4658       case DM_COORD_MAP_FLARE:
4659         mapFunc = coordMap_flare;
4660         if (!Np) {
4661           Np        = cdim + 1;
4662           params[0] = 0;
4663           for (PetscInt d = 1; d <= cdim; ++d) params[d] = 1.0;
4664         }
4665         PetscCheck(Np == cdim + 1, comm, PETSC_ERR_ARG_WRONG, "The flare coordinate map must have cdim + 1 = %" PetscInt_FMT " parameters, not %" PetscInt_FMT, cdim + 1, Np);
4666         break;
4667       case DM_COORD_MAP_ANNULUS:
4668         mapFunc = coordMap_annulus;
4669         if (!Np) {
4670           Np        = 2;
4671           params[0] = 1.;
4672           params[1] = 2.;
4673         }
4674         PetscCheck(Np == 2, comm, PETSC_ERR_ARG_WRONG, "The annulus coordinate map must have 2 parameters, not %" PetscInt_FMT, Np);
4675         break;
4676       case DM_COORD_MAP_SHELL:
4677         mapFunc = coordMap_shell;
4678         if (!Np) {
4679           Np        = 2;
4680           params[0] = 1.;
4681           params[1] = 2.;
4682         }
4683         PetscCheck(Np == 2, comm, PETSC_ERR_ARG_WRONG, "The spherical shell coordinate map must have 2 parameters, not %" PetscInt_FMT, Np);
4684         break;
4685       default:
4686         mapFunc = coordMap_identity;
4687       }
4688     }
4689     if (Np) {
4690       DM      cdm;
4691       PetscDS cds;
4692 
4693       PetscCall(DMGetCoordinateDM(dm, &cdm));
4694       PetscCall(DMGetDS(cdm, &cds));
4695       PetscCall(PetscDSSetConstants(cds, Np, params));
4696     }
4697     PetscCall(DMPlexRemapGeometry(dm, 0.0, mapFunc));
4698   }
4699   /* Handle ghost cells */
4700   PetscCall(PetscOptionsBool("-dm_plex_create_fv_ghost_cells", "Flag to create finite volume ghost cells on the boundary", "DMCreate", ghostCells, &ghostCells, NULL));
4701   if (ghostCells) {
4702     DM   gdm;
4703     char lname[PETSC_MAX_PATH_LEN];
4704 
4705     lname[0] = '\0';
4706     PetscCall(PetscOptionsString("-dm_plex_fv_ghost_cells_label", "Label name for ghost cells boundary", "DMCreate", lname, lname, sizeof(lname), &flg));
4707     PetscCall(DMPlexConstructGhostCells(dm, flg ? lname : NULL, NULL, &gdm));
4708     PetscCall(DMPlexReplace_Internal(dm, &gdm));
4709   }
4710   /* Handle 1D order */
4711   if (reorder != DM_REORDER_DEFAULT_FALSE && dim == 1) {
4712     DM           cdm, rdm;
4713     PetscDS      cds;
4714     PetscObject  obj;
4715     PetscClassId id = PETSC_OBJECT_CLASSID;
4716     IS           perm;
4717     PetscInt     Nf;
4718     PetscBool    distributed;
4719 
4720     PetscCall(DMPlexIsDistributed(dm, &distributed));
4721     PetscCall(DMGetCoordinateDM(dm, &cdm));
4722     PetscCall(DMGetDS(cdm, &cds));
4723     PetscCall(PetscDSGetNumFields(cds, &Nf));
4724     if (Nf) {
4725       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
4726       PetscCall(PetscObjectGetClassId(obj, &id));
4727     }
4728     if (!distributed && id != PETSCFE_CLASSID) {
4729       PetscCall(DMPlexGetOrdering1D(dm, &perm));
4730       PetscCall(DMPlexPermute(dm, perm, &rdm));
4731       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4732       PetscCall(ISDestroy(&perm));
4733     }
4734   }
4735 /* Handle */
4736 non_refine:
4737   PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4738   PetscOptionsHeadEnd();
4739   PetscFunctionReturn(PETSC_SUCCESS);
4740 }
4741 
4742 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm, Vec *vec)
4743 {
4744   PetscFunctionBegin;
4745   PetscCall(DMCreateGlobalVector_Section_Private(dm, vec));
4746   /* PetscCall(VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM)); */
4747   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex));
4748   PetscCall(VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void))VecView_Plex_Native));
4749   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex));
4750   PetscCall(VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void))VecLoad_Plex_Native));
4751   PetscFunctionReturn(PETSC_SUCCESS);
4752 }
4753 
4754 static PetscErrorCode DMCreateLocalVector_Plex(DM dm, Vec *vec)
4755 {
4756   PetscFunctionBegin;
4757   PetscCall(DMCreateLocalVector_Section_Private(dm, vec));
4758   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex_Local));
4759   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex_Local));
4760   PetscFunctionReturn(PETSC_SUCCESS);
4761 }
4762 
4763 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
4764 {
4765   PetscInt depth, d;
4766 
4767   PetscFunctionBegin;
4768   PetscCall(DMPlexGetDepth(dm, &depth));
4769   if (depth == 1) {
4770     PetscCall(DMGetDimension(dm, &d));
4771     if (dim == 0) PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4772     else if (dim == d) PetscCall(DMPlexGetDepthStratum(dm, 1, pStart, pEnd));
4773     else {
4774       *pStart = 0;
4775       *pEnd   = 0;
4776     }
4777   } else {
4778     PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4779   }
4780   PetscFunctionReturn(PETSC_SUCCESS);
4781 }
4782 
4783 static PetscErrorCode DMGetNeighbors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
4784 {
4785   PetscSF            sf;
4786   PetscInt           niranks, njranks, n;
4787   const PetscMPIInt *iranks, *jranks;
4788   DM_Plex           *data = (DM_Plex *)dm->data;
4789 
4790   PetscFunctionBegin;
4791   PetscCall(DMGetPointSF(dm, &sf));
4792   if (!data->neighbors) {
4793     PetscCall(PetscSFSetUp(sf));
4794     PetscCall(PetscSFGetRootRanks(sf, &njranks, &jranks, NULL, NULL, NULL));
4795     PetscCall(PetscSFGetLeafRanks(sf, &niranks, &iranks, NULL, NULL));
4796     PetscCall(PetscMalloc1(njranks + niranks + 1, &data->neighbors));
4797     PetscCall(PetscArraycpy(data->neighbors + 1, jranks, njranks));
4798     PetscCall(PetscArraycpy(data->neighbors + njranks + 1, iranks, niranks));
4799     n = njranks + niranks;
4800     PetscCall(PetscSortRemoveDupsMPIInt(&n, data->neighbors + 1));
4801     /* The following cast should never fail: can't have more neighbors than PETSC_MPI_INT_MAX */
4802     PetscCall(PetscMPIIntCast(n, data->neighbors));
4803   }
4804   if (nranks) *nranks = data->neighbors[0];
4805   if (ranks) {
4806     if (data->neighbors[0]) *ranks = data->neighbors + 1;
4807     else *ranks = NULL;
4808   }
4809   PetscFunctionReturn(PETSC_SUCCESS);
4810 }
4811 
4812 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM, DM, Mat, Vec, Vec);
4813 
4814 static PetscErrorCode DMInitialize_Plex(DM dm)
4815 {
4816   PetscFunctionBegin;
4817   dm->ops->view                      = DMView_Plex;
4818   dm->ops->load                      = DMLoad_Plex;
4819   dm->ops->setfromoptions            = DMSetFromOptions_Plex;
4820   dm->ops->clone                     = DMClone_Plex;
4821   dm->ops->setup                     = DMSetUp_Plex;
4822   dm->ops->createlocalsection        = DMCreateLocalSection_Plex;
4823   dm->ops->createsectionpermutation  = DMCreateSectionPermutation_Plex;
4824   dm->ops->createdefaultconstraints  = DMCreateDefaultConstraints_Plex;
4825   dm->ops->createglobalvector        = DMCreateGlobalVector_Plex;
4826   dm->ops->createlocalvector         = DMCreateLocalVector_Plex;
4827   dm->ops->getlocaltoglobalmapping   = NULL;
4828   dm->ops->createfieldis             = NULL;
4829   dm->ops->createcoordinatedm        = DMCreateCoordinateDM_Plex;
4830   dm->ops->createcoordinatefield     = DMCreateCoordinateField_Plex;
4831   dm->ops->getcoloring               = NULL;
4832   dm->ops->creatematrix              = DMCreateMatrix_Plex;
4833   dm->ops->createinterpolation       = DMCreateInterpolation_Plex;
4834   dm->ops->createmassmatrix          = DMCreateMassMatrix_Plex;
4835   dm->ops->createmassmatrixlumped    = DMCreateMassMatrixLumped_Plex;
4836   dm->ops->createinjection           = DMCreateInjection_Plex;
4837   dm->ops->refine                    = DMRefine_Plex;
4838   dm->ops->coarsen                   = DMCoarsen_Plex;
4839   dm->ops->refinehierarchy           = DMRefineHierarchy_Plex;
4840   dm->ops->coarsenhierarchy          = DMCoarsenHierarchy_Plex;
4841   dm->ops->extrude                   = DMExtrude_Plex;
4842   dm->ops->globaltolocalbegin        = NULL;
4843   dm->ops->globaltolocalend          = NULL;
4844   dm->ops->localtoglobalbegin        = NULL;
4845   dm->ops->localtoglobalend          = NULL;
4846   dm->ops->destroy                   = DMDestroy_Plex;
4847   dm->ops->createsubdm               = DMCreateSubDM_Plex;
4848   dm->ops->createsuperdm             = DMCreateSuperDM_Plex;
4849   dm->ops->getdimpoints              = DMGetDimPoints_Plex;
4850   dm->ops->locatepoints              = DMLocatePoints_Plex;
4851   dm->ops->projectfunctionlocal      = DMProjectFunctionLocal_Plex;
4852   dm->ops->projectfunctionlabellocal = DMProjectFunctionLabelLocal_Plex;
4853   dm->ops->projectfieldlocal         = DMProjectFieldLocal_Plex;
4854   dm->ops->projectfieldlabellocal    = DMProjectFieldLabelLocal_Plex;
4855   dm->ops->projectbdfieldlabellocal  = DMProjectBdFieldLabelLocal_Plex;
4856   dm->ops->computel2diff             = DMComputeL2Diff_Plex;
4857   dm->ops->computel2gradientdiff     = DMComputeL2GradientDiff_Plex;
4858   dm->ops->computel2fielddiff        = DMComputeL2FieldDiff_Plex;
4859   dm->ops->getneighbors              = DMGetNeighbors_Plex;
4860   dm->ops->getlocalboundingbox       = DMGetLocalBoundingBox_Coordinates;
4861   dm->ops->createdomaindecomposition = DMCreateDomainDecomposition_Plex;
4862   dm->ops->createddscatters          = DMCreateDomainDecompositionScatters_Plex;
4863   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertBoundaryValues_C", DMPlexInsertBoundaryValues_Plex));
4864   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertTimeDerivativeBoundaryValues_C", DMPlexInsertTimeDerivativeBoundaryValues_Plex));
4865   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMSetUpGLVisViewer_C", DMSetUpGLVisViewer_Plex));
4866   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMCreateNeumannOverlap_C", DMCreateNeumannOverlap_Plex));
4867   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeGetDefault_C", DMPlexDistributeGetDefault_Plex));
4868   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeSetDefault_C", DMPlexDistributeSetDefault_Plex));
4869   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderGetDefault_C", DMPlexReorderGetDefault_Plex));
4870   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderSetDefault_C", DMPlexReorderSetDefault_Plex));
4871   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionGetDefault_C", DMReorderSectionGetDefault_Plex));
4872   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionSetDefault_C", DMReorderSectionSetDefault_Plex));
4873   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionGetType_C", DMReorderSectionGetType_Plex));
4874   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionSetType_C", DMReorderSectionSetType_Plex));
4875   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMInterpolateSolution_C", DMInterpolateSolution_Plex));
4876   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4877   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetOverlap_C", DMPlexSetOverlap_Plex));
4878   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetUseCeed_C", DMPlexGetUseCeed_Plex));
4879   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetUseCeed_C", DMPlexSetUseCeed_Plex));
4880   PetscFunctionReturn(PETSC_SUCCESS);
4881 }
4882 
4883 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
4884 {
4885   DM_Plex       *mesh = (DM_Plex *)dm->data;
4886   const PetscSF *face_sfs;
4887   PetscInt       num_face_sfs;
4888 
4889   PetscFunctionBegin;
4890   mesh->refct++;
4891   (*newdm)->data = mesh;
4892   PetscCall(DMPlexGetIsoperiodicFaceSF(dm, &num_face_sfs, &face_sfs));
4893   PetscCall(DMPlexSetIsoperiodicFaceSF(*newdm, num_face_sfs, (PetscSF *)face_sfs));
4894   PetscCall(PetscObjectChangeTypeName((PetscObject)*newdm, DMPLEX));
4895   PetscCall(DMInitialize_Plex(*newdm));
4896   PetscFunctionReturn(PETSC_SUCCESS);
4897 }
4898 
4899 /*MC
4900   DMPLEX = "plex" - A `DM` object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
4901                     In the local representation, `Vec`s contain all unknowns in the interior and shared boundary. This is
4902                     specified by a PetscSection object. Ownership in the global representation is determined by
4903                     ownership of the underlying `DMPLEX` points. This is specified by another `PetscSection` object.
4904 
4905   Options Database Keys:
4906 + -dm_refine_pre                     - Refine mesh before distribution
4907 + -dm_refine_uniform_pre             - Choose uniform or generator-based refinement
4908 + -dm_refine_volume_limit_pre        - Cell volume limit after pre-refinement using generator
4909 . -dm_distribute                     - Distribute mesh across processes
4910 . -dm_distribute_overlap             - Number of cells to overlap for distribution
4911 . -dm_refine                         - Refine mesh after distribution
4912 . -dm_localize <bool>                - Whether to localize coordinates for periodic meshes
4913 . -dm_sparse_localize <bool>         - Whether to only localize cells on the periodic boundary
4914 . -dm_plex_hash_location             - Use grid hashing for point location
4915 . -dm_plex_hash_box_faces <n,m,p>    - The number of divisions in each direction of the grid hash
4916 . -dm_plex_partition_balance         - Attempt to evenly divide points on partition boundary between processes
4917 . -dm_plex_remesh_bd                 - Allow changes to the boundary on remeshing
4918 . -dm_plex_max_projection_height     - Maximum mesh point height used to project locally
4919 . -dm_plex_regular_refinement        - Use special nested projection algorithm for regular refinement
4920 . -dm_plex_reorder_section           - Use specialized blocking if available
4921 . -dm_plex_check_all                 - Perform all checks below
4922 . -dm_plex_check_symmetry            - Check that the adjacency information in the mesh is symmetric
4923 . -dm_plex_check_skeleton <celltype> - Check that each cell has the correct number of vertices
4924 . -dm_plex_check_faces <celltype>    - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
4925 . -dm_plex_check_geometry            - Check that cells have positive volume
4926 . -dm_view :mesh.tex:ascii_latex     - View the mesh in LaTeX/TikZ
4927 . -dm_plex_view_scale <num>          - Scale the TikZ
4928 . -dm_plex_print_fem <num>           - View FEM assembly information, such as element vectors and matrices
4929 - -dm_plex_print_fvm <num>           - View FVM assembly information, such as flux updates
4930 
4931   Level: intermediate
4932 
4933 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`, `PetscSection`
4934 M*/
4935 
4936 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
4937 {
4938   DM_Plex *mesh;
4939   PetscInt unit;
4940 
4941   PetscFunctionBegin;
4942   PetscCall(PetscCitationsRegister(PlexCitation, &Plexcite));
4943   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4944   PetscCall(PetscNew(&mesh));
4945   dm->reorderSection = DM_REORDER_DEFAULT_NOTSET;
4946   dm->data           = mesh;
4947 
4948   mesh->refct = 1;
4949   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection));
4950   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection));
4951   mesh->refinementUniform      = PETSC_TRUE;
4952   mesh->refinementLimit        = -1.0;
4953   mesh->distDefault            = PETSC_TRUE;
4954   mesh->reorderDefault         = DM_REORDER_DEFAULT_NOTSET;
4955   mesh->distributionName       = NULL;
4956   mesh->interpolated           = DMPLEX_INTERPOLATED_INVALID;
4957   mesh->interpolatedCollective = DMPLEX_INTERPOLATED_INVALID;
4958 
4959   PetscCall(PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner));
4960   mesh->remeshBd = PETSC_FALSE;
4961 
4962   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
4963 
4964   mesh->depthState    = -1;
4965   mesh->celltypeState = -1;
4966   mesh->printTol      = 1.0e-10;
4967 
4968   PetscCall(DMInitialize_Plex(dm));
4969   PetscFunctionReturn(PETSC_SUCCESS);
4970 }
4971 
4972 /*@
4973   DMPlexCreate - Creates a `DMPLEX` object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
4974 
4975   Collective
4976 
4977   Input Parameter:
4978 . comm - The communicator for the `DMPLEX` object
4979 
4980   Output Parameter:
4981 . mesh - The `DMPLEX` object
4982 
4983   Level: beginner
4984 
4985 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMType`, `DMCreate()`, `DMSetType()`
4986 @*/
4987 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
4988 {
4989   PetscFunctionBegin;
4990   PetscAssertPointer(mesh, 2);
4991   PetscCall(DMCreate(comm, mesh));
4992   PetscCall(DMSetType(*mesh, DMPLEX));
4993   PetscFunctionReturn(PETSC_SUCCESS);
4994 }
4995 
4996 /*@C
4997   DMPlexBuildFromCellListParallel - Build distributed `DMPLEX` topology from a list of vertices for each cell (common mesh generator output)
4998 
4999   Collective; No Fortran Support
5000 
5001   Input Parameters:
5002 + dm          - The `DM`
5003 . numCells    - The number of cells owned by this process
5004 . numVertices - The number of vertices to be owned by this process, or `PETSC_DECIDE`
5005 . NVertices   - The global number of vertices, or `PETSC_DETERMINE`
5006 . numCorners  - The number of vertices for each cell
5007 - cells       - An array of numCells*numCorners numbers, the global vertex numbers for each cell
5008 
5009   Output Parameters:
5010 + vertexSF         - (Optional) `PetscSF` describing complete vertex ownership
5011 - verticesAdjSaved - (Optional) vertex adjacency array
5012 
5013   Level: advanced
5014 
5015   Notes:
5016   Two triangles sharing a face
5017 .vb
5018 
5019         2
5020       / | \
5021      /  |  \
5022     /   |   \
5023    0  0 | 1  3
5024     \   |   /
5025      \  |  /
5026       \ | /
5027         1
5028 .ve
5029   would have input
5030 .vb
5031   numCells = 2, numVertices = 4
5032   cells = [0 1 2  1 3 2]
5033 .ve
5034   which would result in the `DMPLEX`
5035 .vb
5036 
5037         4
5038       / | \
5039      /  |  \
5040     /   |   \
5041    2  0 | 1  5
5042     \   |   /
5043      \  |  /
5044       \ | /
5045         3
5046 .ve
5047 
5048   Vertices are implicitly numbered consecutively 0,...,NVertices.
5049   Each rank owns a chunk of numVertices consecutive vertices.
5050   If numVertices is `PETSC_DECIDE`, PETSc will distribute them as evenly as possible using PetscLayout.
5051   If NVertices is `PETSC_DETERMINE` and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
5052   If only NVertices is `PETSC_DETERMINE`, it is computed as the sum of numVertices over all ranks.
5053 
5054   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
5055 
5056 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildCoordinatesFromCellListParallel()`,
5057           `PetscSF`
5058 @*/
5059 PetscErrorCode DMPlexBuildFromCellListParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, const PetscInt cells[], PetscSF *vertexSF, PetscInt **verticesAdjSaved)
5060 {
5061   PetscSF     sfPoint;
5062   PetscLayout layout;
5063   PetscInt    numVerticesAdj, *verticesAdj, *cones, c, p;
5064 
5065   PetscFunctionBegin;
5066   PetscValidLogicalCollectiveInt(dm, NVertices, 4);
5067   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5068   /* Get/check global number of vertices */
5069   {
5070     PetscInt       NVerticesInCells, i;
5071     const PetscInt len = numCells * numCorners;
5072 
5073     /* NVerticesInCells = max(cells) + 1 */
5074     NVerticesInCells = PETSC_MIN_INT;
5075     for (i = 0; i < len; i++)
5076       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
5077     ++NVerticesInCells;
5078     PetscCall(MPIU_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
5079 
5080     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
5081     else
5082       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
5083   }
5084   /* Count locally unique vertices */
5085   {
5086     PetscHSetI vhash;
5087     PetscInt   off = 0;
5088 
5089     PetscCall(PetscHSetICreate(&vhash));
5090     for (c = 0; c < numCells; ++c) {
5091       for (p = 0; p < numCorners; ++p) PetscCall(PetscHSetIAdd(vhash, cells[c * numCorners + p]));
5092     }
5093     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
5094     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
5095     else verticesAdj = *verticesAdjSaved;
5096     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
5097     PetscCall(PetscHSetIDestroy(&vhash));
5098     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
5099   }
5100   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
5101   /* Create cones */
5102   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
5103   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
5104   PetscCall(DMSetUp(dm));
5105   PetscCall(DMPlexGetCones(dm, &cones));
5106   for (c = 0; c < numCells; ++c) {
5107     for (p = 0; p < numCorners; ++p) {
5108       const PetscInt gv = cells[c * numCorners + p];
5109       PetscInt       lv;
5110 
5111       /* Positions within verticesAdj form 0-based local vertex numbering;
5112          we need to shift it by numCells to get correct DAG points (cells go first) */
5113       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
5114       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
5115       cones[c * numCorners + p] = lv + numCells;
5116     }
5117   }
5118   /* Build point sf */
5119   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
5120   PetscCall(PetscLayoutSetSize(layout, NVertices));
5121   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
5122   PetscCall(PetscLayoutSetBlockSize(layout, 1));
5123   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
5124   PetscCall(PetscLayoutDestroy(&layout));
5125   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
5126   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
5127   if (dm->sf) {
5128     const char *prefix;
5129 
5130     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
5131     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
5132   }
5133   PetscCall(DMSetPointSF(dm, sfPoint));
5134   PetscCall(PetscSFDestroy(&sfPoint));
5135   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)*vertexSF, "Vertex Ownership SF"));
5136   /* Fill in the rest of the topology structure */
5137   PetscCall(DMPlexSymmetrize(dm));
5138   PetscCall(DMPlexStratify(dm));
5139   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5140   PetscFunctionReturn(PETSC_SUCCESS);
5141 }
5142 
5143 /*@
5144   DMPlexBuildCoordinatesFromCellListParallel - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
5145 
5146   Collective; No Fortran Support
5147 
5148   Input Parameters:
5149 + dm           - The `DM`
5150 . spaceDim     - The spatial dimension used for coordinates
5151 . sfVert       - `PetscSF` describing complete vertex ownership
5152 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
5153 
5154   Level: advanced
5155 
5156 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellListParallel()`
5157 @*/
5158 PetscErrorCode DMPlexBuildCoordinatesFromCellListParallel(DM dm, PetscInt spaceDim, PetscSF sfVert, const PetscReal vertexCoords[])
5159 {
5160   PetscSection coordSection;
5161   Vec          coordinates;
5162   PetscScalar *coords;
5163   PetscInt     numVertices, numVerticesAdj, coordSize, v, vStart, vEnd;
5164 
5165   PetscFunctionBegin;
5166   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
5167   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
5168   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
5169   PetscCall(DMSetCoordinateDim(dm, spaceDim));
5170   PetscCall(PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL));
5171   PetscCheck(vEnd - vStart == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Supplied sfVert has wrong number of leaves = %" PetscInt_FMT " != %" PetscInt_FMT " = vEnd - vStart", numVerticesAdj, vEnd - vStart);
5172   PetscCall(DMGetCoordinateSection(dm, &coordSection));
5173   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5174   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
5175   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
5176   for (v = vStart; v < vEnd; ++v) {
5177     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
5178     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
5179   }
5180   PetscCall(PetscSectionSetUp(coordSection));
5181   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
5182   PetscCall(VecCreate(PetscObjectComm((PetscObject)dm), &coordinates));
5183   PetscCall(VecSetBlockSize(coordinates, spaceDim));
5184   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5185   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
5186   PetscCall(VecSetType(coordinates, VECSTANDARD));
5187   PetscCall(VecGetArray(coordinates, &coords));
5188   {
5189     MPI_Datatype coordtype;
5190 
5191     /* Need a temp buffer for coords if we have complex/single */
5192     PetscCallMPI(MPI_Type_contiguous(spaceDim, MPIU_SCALAR, &coordtype));
5193     PetscCallMPI(MPI_Type_commit(&coordtype));
5194 #if defined(PETSC_USE_COMPLEX)
5195     {
5196       PetscScalar *svertexCoords;
5197       PetscInt     i;
5198       PetscCall(PetscMalloc1(numVertices * spaceDim, &svertexCoords));
5199       for (i = 0; i < numVertices * spaceDim; i++) svertexCoords[i] = vertexCoords[i];
5200       PetscCall(PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
5201       PetscCall(PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
5202       PetscCall(PetscFree(svertexCoords));
5203     }
5204 #else
5205     PetscCall(PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
5206     PetscCall(PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
5207 #endif
5208     PetscCallMPI(MPI_Type_free(&coordtype));
5209   }
5210   PetscCall(VecRestoreArray(coordinates, &coords));
5211   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
5212   PetscCall(VecDestroy(&coordinates));
5213   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
5214   PetscFunctionReturn(PETSC_SUCCESS);
5215 }
5216 
5217 /*@
5218   DMPlexCreateFromCellListParallelPetsc - Create distributed `DMPLEX` from a list of vertices for each cell (common mesh generator output)
5219 
5220   Collective
5221 
5222   Input Parameters:
5223 + comm         - The communicator
5224 . dim          - The topological dimension of the mesh
5225 . numCells     - The number of cells owned by this process
5226 . numVertices  - The number of vertices owned by this process, or `PETSC_DECIDE`
5227 . NVertices    - The global number of vertices, or `PETSC_DECIDE`
5228 . numCorners   - The number of vertices for each cell
5229 . interpolate  - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
5230 . cells        - An array of numCells*numCorners numbers, the global vertex numbers for each cell
5231 . spaceDim     - The spatial dimension used for coordinates
5232 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
5233 
5234   Output Parameters:
5235 + dm          - The `DM`
5236 . vertexSF    - (Optional) `PetscSF` describing complete vertex ownership
5237 - verticesAdj - (Optional) vertex adjacency array
5238 
5239   Level: intermediate
5240 
5241   Notes:
5242   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`,
5243   `DMPlexBuildFromCellListParallel()`, `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellListParallel()`
5244 
5245   See `DMPlexBuildFromCellListParallel()` for an example and details about the topology-related parameters.
5246 
5247   See `DMPlexBuildCoordinatesFromCellListParallel()` for details about the geometry-related parameters.
5248 
5249 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
5250 @*/
5251 PetscErrorCode DMPlexCreateFromCellListParallelPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, PetscInt **verticesAdj, DM *dm)
5252 {
5253   PetscSF sfVert;
5254 
5255   PetscFunctionBegin;
5256   PetscCall(DMCreate(comm, dm));
5257   PetscCall(DMSetType(*dm, DMPLEX));
5258   PetscValidLogicalCollectiveInt(*dm, dim, 2);
5259   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
5260   PetscCall(DMSetDimension(*dm, dim));
5261   PetscCall(DMPlexBuildFromCellListParallel(*dm, numCells, numVertices, NVertices, numCorners, cells, &sfVert, verticesAdj));
5262   if (interpolate) {
5263     DM idm;
5264 
5265     PetscCall(DMPlexInterpolate(*dm, &idm));
5266     PetscCall(DMDestroy(dm));
5267     *dm = idm;
5268   }
5269   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
5270   if (vertexSF) *vertexSF = sfVert;
5271   else PetscCall(PetscSFDestroy(&sfVert));
5272   PetscFunctionReturn(PETSC_SUCCESS);
5273 }
5274 
5275 /*@
5276   DMPlexBuildFromCellList - Build `DMPLEX` topology from a list of vertices for each cell (common mesh generator output)
5277 
5278   Collective; No Fortran Support
5279 
5280   Input Parameters:
5281 + dm          - The `DM`
5282 . numCells    - The number of cells owned by this process
5283 . numVertices - The number of vertices owned by this process, or `PETSC_DETERMINE`
5284 . numCorners  - The number of vertices for each cell
5285 - cells       - An array of `numCells` x `numCorners` numbers, the global vertex numbers for each cell
5286 
5287   Level: advanced
5288 
5289   Notes:
5290   Two triangles sharing a face
5291 .vb
5292 
5293         2
5294       / | \
5295      /  |  \
5296     /   |   \
5297    0  0 | 1  3
5298     \   |   /
5299      \  |  /
5300       \ | /
5301         1
5302 .ve
5303   would have input
5304 .vb
5305   numCells = 2, numVertices = 4
5306   cells = [0 1 2  1 3 2]
5307 .ve
5308   which would result in the `DMPLEX`
5309 .vb
5310 
5311         4
5312       / | \
5313      /  |  \
5314     /   |   \
5315    2  0 | 1  5
5316     \   |   /
5317      \  |  /
5318       \ | /
5319         3
5320 .ve
5321 
5322   If numVertices is `PETSC_DETERMINE`, it is computed by PETSc as the maximum vertex index in cells + 1.
5323 
5324 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListPetsc()`
5325 @*/
5326 PetscErrorCode DMPlexBuildFromCellList(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const PetscInt cells[])
5327 {
5328   PetscInt *cones, c, p, dim;
5329 
5330   PetscFunctionBegin;
5331   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5332   PetscCall(DMGetDimension(dm, &dim));
5333   /* Get/check global number of vertices */
5334   {
5335     PetscInt       NVerticesInCells, i;
5336     const PetscInt len = numCells * numCorners;
5337 
5338     /* NVerticesInCells = max(cells) + 1 */
5339     NVerticesInCells = PETSC_MIN_INT;
5340     for (i = 0; i < len; i++)
5341       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
5342     ++NVerticesInCells;
5343 
5344     if (numVertices == PETSC_DECIDE) numVertices = NVerticesInCells;
5345     else
5346       PetscCheck(numVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, numVertices, NVerticesInCells);
5347   }
5348   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
5349   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
5350   PetscCall(DMSetUp(dm));
5351   PetscCall(DMPlexGetCones(dm, &cones));
5352   for (c = 0; c < numCells; ++c) {
5353     for (p = 0; p < numCorners; ++p) cones[c * numCorners + p] = cells[c * numCorners + p] + numCells;
5354   }
5355   PetscCall(DMPlexSymmetrize(dm));
5356   PetscCall(DMPlexStratify(dm));
5357   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5358   PetscFunctionReturn(PETSC_SUCCESS);
5359 }
5360 
5361 /*@
5362   DMPlexBuildCoordinatesFromCellList - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
5363 
5364   Collective
5365 
5366   Input Parameters:
5367 + dm           - The `DM`
5368 . spaceDim     - The spatial dimension used for coordinates
5369 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
5370 
5371   Level: advanced
5372 
5373 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellList()`
5374 @*/
5375 PetscErrorCode DMPlexBuildCoordinatesFromCellList(DM dm, PetscInt spaceDim, const PetscReal vertexCoords[])
5376 {
5377   PetscSection coordSection;
5378   Vec          coordinates;
5379   DM           cdm;
5380   PetscScalar *coords;
5381   PetscInt     v, vStart, vEnd, d;
5382 
5383   PetscFunctionBegin;
5384   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
5385   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
5386   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
5387   PetscCall(DMSetCoordinateDim(dm, spaceDim));
5388   PetscCall(DMGetCoordinateSection(dm, &coordSection));
5389   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5390   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
5391   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
5392   for (v = vStart; v < vEnd; ++v) {
5393     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
5394     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
5395   }
5396   PetscCall(PetscSectionSetUp(coordSection));
5397 
5398   PetscCall(DMGetCoordinateDM(dm, &cdm));
5399   PetscCall(DMCreateLocalVector(cdm, &coordinates));
5400   PetscCall(VecSetBlockSize(coordinates, spaceDim));
5401   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5402   PetscCall(VecGetArrayWrite(coordinates, &coords));
5403   for (v = 0; v < vEnd - vStart; ++v) {
5404     for (d = 0; d < spaceDim; ++d) coords[v * spaceDim + d] = vertexCoords[v * spaceDim + d];
5405   }
5406   PetscCall(VecRestoreArrayWrite(coordinates, &coords));
5407   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
5408   PetscCall(VecDestroy(&coordinates));
5409   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
5410   PetscFunctionReturn(PETSC_SUCCESS);
5411 }
5412 
5413 /*@
5414   DMPlexCreateFromCellListPetsc - Create `DMPLEX` from a list of vertices for each cell (common mesh generator output), but only process 0 takes in the input
5415 
5416   Collective
5417 
5418   Input Parameters:
5419 + comm         - The communicator
5420 . dim          - The topological dimension of the mesh
5421 . numCells     - The number of cells, only on process 0
5422 . numVertices  - The number of vertices owned by this process, or `PETSC_DECIDE`, only on process 0
5423 . numCorners   - The number of vertices for each cell, only on process 0
5424 . interpolate  - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
5425 . cells        - An array of numCells*numCorners numbers, the vertices for each cell, only on process 0
5426 . spaceDim     - The spatial dimension used for coordinates
5427 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex, only on process 0
5428 
5429   Output Parameter:
5430 . dm - The `DM`, which only has points on process 0
5431 
5432   Level: intermediate
5433 
5434   Notes:
5435   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`, `DMPlexBuildFromCellList()`,
5436   `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellList()`
5437 
5438   See `DMPlexBuildFromCellList()` for an example and details about the topology-related parameters.
5439   See `DMPlexBuildCoordinatesFromCellList()` for details about the geometry-related parameters.
5440   See `DMPlexCreateFromCellListParallelPetsc()` for parallel input
5441 
5442 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellList()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
5443 @*/
5444 PetscErrorCode DMPlexCreateFromCellListPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], DM *dm)
5445 {
5446   PetscMPIInt rank;
5447 
5448   PetscFunctionBegin;
5449   PetscCheck(dim, comm, PETSC_ERR_ARG_OUTOFRANGE, "This is not appropriate for 0-dimensional meshes. Consider either creating the DM using DMPlexCreateFromDAG(), by hand, or using DMSwarm.");
5450   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5451   PetscCall(DMCreate(comm, dm));
5452   PetscCall(DMSetType(*dm, DMPLEX));
5453   PetscCall(DMSetDimension(*dm, dim));
5454   if (rank == 0) PetscCall(DMPlexBuildFromCellList(*dm, numCells, numVertices, numCorners, cells));
5455   else PetscCall(DMPlexBuildFromCellList(*dm, 0, 0, 0, NULL));
5456   if (interpolate) {
5457     DM idm;
5458 
5459     PetscCall(DMPlexInterpolate(*dm, &idm));
5460     PetscCall(DMDestroy(dm));
5461     *dm = idm;
5462   }
5463   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, vertexCoords));
5464   else PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, NULL));
5465   PetscFunctionReturn(PETSC_SUCCESS);
5466 }
5467 
5468 /*@
5469   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a `DM`
5470 
5471   Input Parameters:
5472 + dm               - The empty `DM` object, usually from `DMCreate()` and `DMSetDimension()`
5473 . depth            - The depth of the DAG
5474 . numPoints        - Array of size depth + 1 containing the number of points at each `depth`
5475 . coneSize         - The cone size of each point
5476 . cones            - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
5477 . coneOrientations - The orientation of each cone point
5478 - vertexCoords     - An array of `numPoints`[0]*spacedim numbers representing the coordinates of each vertex, with spacedim the value set via `DMSetCoordinateDim()`
5479 
5480   Output Parameter:
5481 . dm - The `DM`
5482 
5483   Level: advanced
5484 
5485   Note:
5486   Two triangles sharing a face would have input
5487 .vb
5488   depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
5489   cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
5490  vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
5491 .ve
5492   which would result in the DMPlex
5493 .vb
5494         4
5495       / | \
5496      /  |  \
5497     /   |   \
5498    2  0 | 1  5
5499     \   |   /
5500      \  |  /
5501       \ | /
5502         3
5503 .ve
5504   Notice that all points are numbered consecutively, unlike `DMPlexCreateFromCellListPetsc()`
5505 
5506 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
5507 @*/
5508 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
5509 {
5510   Vec          coordinates;
5511   PetscSection coordSection;
5512   PetscScalar *coords;
5513   PetscInt     coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
5514 
5515   PetscFunctionBegin;
5516   PetscCall(DMGetDimension(dm, &dim));
5517   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
5518   PetscCheck(dimEmbed >= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Embedding dimension %" PetscInt_FMT " cannot be less than intrinsic dimension %" PetscInt_FMT, dimEmbed, dim);
5519   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
5520   PetscCall(DMPlexSetChart(dm, pStart, pEnd));
5521   for (p = pStart; p < pEnd; ++p) {
5522     PetscCall(DMPlexSetConeSize(dm, p, coneSize[p - pStart]));
5523     if (firstVertex < 0 && !coneSize[p - pStart]) firstVertex = p - pStart;
5524   }
5525   PetscCheck(firstVertex >= 0 || !numPoints[0], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Expected %" PetscInt_FMT " vertices but could not find any", numPoints[0]);
5526   PetscCall(DMSetUp(dm)); /* Allocate space for cones */
5527   for (p = pStart, off = 0; p < pEnd; off += coneSize[p - pStart], ++p) {
5528     PetscCall(DMPlexSetCone(dm, p, &cones[off]));
5529     PetscCall(DMPlexSetConeOrientation(dm, p, &coneOrientations[off]));
5530   }
5531   PetscCall(DMPlexSymmetrize(dm));
5532   PetscCall(DMPlexStratify(dm));
5533   /* Build coordinates */
5534   PetscCall(DMGetCoordinateSection(dm, &coordSection));
5535   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5536   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dimEmbed));
5537   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numPoints[0]));
5538   for (v = firstVertex; v < firstVertex + numPoints[0]; ++v) {
5539     PetscCall(PetscSectionSetDof(coordSection, v, dimEmbed));
5540     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed));
5541   }
5542   PetscCall(PetscSectionSetUp(coordSection));
5543   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
5544   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
5545   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5546   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
5547   PetscCall(VecSetBlockSize(coordinates, dimEmbed));
5548   PetscCall(VecSetType(coordinates, VECSTANDARD));
5549   if (vertexCoords) {
5550     PetscCall(VecGetArray(coordinates, &coords));
5551     for (v = 0; v < numPoints[0]; ++v) {
5552       PetscInt off;
5553 
5554       PetscCall(PetscSectionGetOffset(coordSection, v + firstVertex, &off));
5555       for (d = 0; d < dimEmbed; ++d) coords[off + d] = vertexCoords[v * dimEmbed + d];
5556     }
5557   }
5558   PetscCall(VecRestoreArray(coordinates, &coords));
5559   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
5560   PetscCall(VecDestroy(&coordinates));
5561   PetscFunctionReturn(PETSC_SUCCESS);
5562 }
5563 
5564 /*
5565   DMPlexCreateCellVertexFromFile - Create a `DMPLEX` mesh from a simple cell-vertex file.
5566 
5567   Collective
5568 
5569 + comm        - The MPI communicator
5570 . filename    - Name of the .dat file
5571 - interpolate - Create faces and edges in the mesh
5572 
5573   Output Parameter:
5574 . dm  - The `DM` object representing the mesh
5575 
5576   Level: beginner
5577 
5578   Note:
5579   The format is the simplest possible:
5580 .vb
5581   dim Ne Nv Nc Nl
5582   v_1 v_2 ... v_Nc
5583   ...
5584   x y z marker_1 ... marker_Nl
5585 .ve
5586 
5587   Developer Note:
5588   Should use a `PetscViewer` not a filename
5589 
5590 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromFile()`, `DMPlexCreateGmsh()`, `DMPlexCreate()`
5591 */
5592 static PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
5593 {
5594   DMLabel      marker;
5595   PetscViewer  viewer;
5596   Vec          coordinates;
5597   PetscSection coordSection;
5598   PetscScalar *coords;
5599   char         line[PETSC_MAX_PATH_LEN];
5600   PetscInt     cdim, coordSize, v, c, d;
5601   PetscMPIInt  rank;
5602   int          snum, dim, Nv, Nc, Ncn, Nl;
5603 
5604   PetscFunctionBegin;
5605   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5606   PetscCall(PetscViewerCreate(comm, &viewer));
5607   PetscCall(PetscViewerSetType(viewer, PETSCVIEWERASCII));
5608   PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
5609   PetscCall(PetscViewerFileSetName(viewer, filename));
5610   if (rank == 0) {
5611     PetscCall(PetscViewerRead(viewer, line, 5, NULL, PETSC_STRING));
5612     snum = sscanf(line, "%d %d %d %d %d", &dim, &Nc, &Nv, &Ncn, &Nl);
5613     PetscCheck(snum == 5, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5614   } else {
5615     Nc = Nv = Ncn = Nl = 0;
5616   }
5617   PetscCallMPI(MPI_Bcast(&dim, 1, MPI_INT, 0, comm));
5618   cdim = (PetscInt)dim;
5619   PetscCall(DMCreate(comm, dm));
5620   PetscCall(DMSetType(*dm, DMPLEX));
5621   PetscCall(DMPlexSetChart(*dm, 0, Nc + Nv));
5622   PetscCall(DMSetDimension(*dm, (PetscInt)dim));
5623   PetscCall(DMSetCoordinateDim(*dm, cdim));
5624   /* Read topology */
5625   if (rank == 0) {
5626     char     format[PETSC_MAX_PATH_LEN];
5627     PetscInt cone[8];
5628     int      vbuf[8], v;
5629 
5630     for (c = 0; c < Ncn; ++c) {
5631       format[c * 3 + 0] = '%';
5632       format[c * 3 + 1] = 'd';
5633       format[c * 3 + 2] = ' ';
5634     }
5635     format[Ncn * 3 - 1] = '\0';
5636     for (c = 0; c < Nc; ++c) PetscCall(DMPlexSetConeSize(*dm, c, Ncn));
5637     PetscCall(DMSetUp(*dm));
5638     for (c = 0; c < Nc; ++c) {
5639       PetscCall(PetscViewerRead(viewer, line, Ncn, NULL, PETSC_STRING));
5640       switch (Ncn) {
5641       case 2:
5642         snum = sscanf(line, format, &vbuf[0], &vbuf[1]);
5643         break;
5644       case 3:
5645         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2]);
5646         break;
5647       case 4:
5648         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3]);
5649         break;
5650       case 6:
5651         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5]);
5652         break;
5653       case 8:
5654         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);
5655         break;
5656       default:
5657         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No cell shape with %d vertices", Ncn);
5658       }
5659       PetscCheck(snum == Ncn, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5660       for (v = 0; v < Ncn; ++v) cone[v] = vbuf[v] + Nc;
5661       /* Hexahedra are inverted */
5662       if (Ncn == 8) {
5663         PetscInt tmp = cone[1];
5664         cone[1]      = cone[3];
5665         cone[3]      = tmp;
5666       }
5667       PetscCall(DMPlexSetCone(*dm, c, cone));
5668     }
5669   }
5670   PetscCall(DMPlexSymmetrize(*dm));
5671   PetscCall(DMPlexStratify(*dm));
5672   /* Read coordinates */
5673   PetscCall(DMGetCoordinateSection(*dm, &coordSection));
5674   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5675   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
5676   PetscCall(PetscSectionSetChart(coordSection, Nc, Nc + Nv));
5677   for (v = Nc; v < Nc + Nv; ++v) {
5678     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
5679     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
5680   }
5681   PetscCall(PetscSectionSetUp(coordSection));
5682   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
5683   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
5684   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5685   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
5686   PetscCall(VecSetBlockSize(coordinates, cdim));
5687   PetscCall(VecSetType(coordinates, VECSTANDARD));
5688   PetscCall(VecGetArray(coordinates, &coords));
5689   if (rank == 0) {
5690     char   format[PETSC_MAX_PATH_LEN];
5691     double x[3];
5692     int    l, val[3];
5693 
5694     if (Nl) {
5695       for (l = 0; l < Nl; ++l) {
5696         format[l * 3 + 0] = '%';
5697         format[l * 3 + 1] = 'd';
5698         format[l * 3 + 2] = ' ';
5699       }
5700       format[Nl * 3 - 1] = '\0';
5701       PetscCall(DMCreateLabel(*dm, "marker"));
5702       PetscCall(DMGetLabel(*dm, "marker", &marker));
5703     }
5704     for (v = 0; v < Nv; ++v) {
5705       PetscCall(PetscViewerRead(viewer, line, 3 + Nl, NULL, PETSC_STRING));
5706       snum = sscanf(line, "%lg %lg %lg", &x[0], &x[1], &x[2]);
5707       PetscCheck(snum == 3, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5708       switch (Nl) {
5709       case 0:
5710         snum = 0;
5711         break;
5712       case 1:
5713         snum = sscanf(line, format, &val[0]);
5714         break;
5715       case 2:
5716         snum = sscanf(line, format, &val[0], &val[1]);
5717         break;
5718       case 3:
5719         snum = sscanf(line, format, &val[0], &val[1], &val[2]);
5720         break;
5721       default:
5722         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Request support for %d labels", Nl);
5723       }
5724       PetscCheck(snum == Nl, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5725       for (d = 0; d < cdim; ++d) coords[v * cdim + d] = x[d];
5726       for (l = 0; l < Nl; ++l) PetscCall(DMLabelSetValue(marker, v + Nc, val[l]));
5727     }
5728   }
5729   PetscCall(VecRestoreArray(coordinates, &coords));
5730   PetscCall(DMSetCoordinatesLocal(*dm, coordinates));
5731   PetscCall(VecDestroy(&coordinates));
5732   PetscCall(PetscViewerDestroy(&viewer));
5733   if (interpolate) {
5734     DM      idm;
5735     DMLabel bdlabel;
5736 
5737     PetscCall(DMPlexInterpolate(*dm, &idm));
5738     PetscCall(DMDestroy(dm));
5739     *dm = idm;
5740 
5741     if (!Nl) {
5742       PetscCall(DMCreateLabel(*dm, "marker"));
5743       PetscCall(DMGetLabel(*dm, "marker", &bdlabel));
5744       PetscCall(DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel));
5745       PetscCall(DMPlexLabelComplete(*dm, bdlabel));
5746     }
5747   }
5748   PetscFunctionReturn(PETSC_SUCCESS);
5749 }
5750 
5751 /*@
5752   DMPlexCreateFromFile - This takes a filename and produces a `DM`
5753 
5754   Collective
5755 
5756   Input Parameters:
5757 + comm        - The communicator
5758 . filename    - A file name
5759 . plexname    - The object name of the resulting `DM`, also used for intra-datafile lookup by some formats
5760 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
5761 
5762   Output Parameter:
5763 . dm - The `DM`
5764 
5765   Options Database Key:
5766 . -dm_plex_create_from_hdf5_xdmf - use the `PETSC_VIEWER_HDF5_XDMF` format for reading HDF5
5767 
5768   Use `-dm_plex_create_ prefix` to pass options to the internal `PetscViewer`, e.g.
5769 $ -dm_plex_create_viewer_hdf5_collective
5770 
5771   Level: beginner
5772 
5773   Notes:
5774   Using `PETSCVIEWERHDF5` type with `PETSC_VIEWER_HDF5_PETSC` format, one can save multiple `DMPLEX`
5775   meshes in a single HDF5 file. This in turn requires one to name the `DMPLEX` object with `PetscObjectSetName()`
5776   before saving it with `DMView()` and before loading it with `DMLoad()` for identification of the mesh object.
5777   The input parameter name is thus used to name the `DMPLEX` object when `DMPlexCreateFromFile()` internally
5778   calls `DMLoad()`. Currently, name is ignored for other viewer types and/or formats.
5779 
5780 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`, `PetscObjectSetName()`, `DMView()`, `DMLoad()`
5781 @*/
5782 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], const char plexname[], PetscBool interpolate, DM *dm)
5783 {
5784   const char  extGmsh[]      = ".msh";
5785   const char  extGmsh2[]     = ".msh2";
5786   const char  extGmsh4[]     = ".msh4";
5787   const char  extCGNS[]      = ".cgns";
5788   const char  extExodus[]    = ".exo";
5789   const char  extExodus_e[]  = ".e";
5790   const char  extGenesis[]   = ".gen";
5791   const char  extFluent[]    = ".cas";
5792   const char  extHDF5[]      = ".h5";
5793   const char  extXDMFHDF5[]  = ".xdmf.h5";
5794   const char  extPLY[]       = ".ply";
5795   const char  extEGADSLite[] = ".egadslite";
5796   const char  extEGADS[]     = ".egads";
5797   const char  extIGES[]      = ".igs";
5798   const char  extSTEP[]      = ".stp";
5799   const char  extCV[]        = ".dat";
5800   size_t      len;
5801   PetscBool   isGmsh, isGmsh2, isGmsh4, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isPLY, isEGADSLite, isEGADS, isIGES, isSTEP, isCV, isXDMFHDF5;
5802   PetscMPIInt rank;
5803 
5804   PetscFunctionBegin;
5805   PetscAssertPointer(filename, 2);
5806   if (plexname) PetscAssertPointer(plexname, 3);
5807   PetscAssertPointer(dm, 5);
5808   PetscCall(DMInitializePackage());
5809   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5810   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5811   PetscCall(PetscStrlen(filename, &len));
5812   PetscCheck(len, comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
5813 
5814 #define CheckExtension(extension__, is_extension__) \
5815   do { \
5816     PetscAssert(sizeof(extension__), comm, PETSC_ERR_PLIB, "Zero-size extension: %s", extension__); \
5817     /* don't count the null-terminator at the end */ \
5818     const size_t ext_len = sizeof(extension__) - 1; \
5819     if (len < ext_len) { \
5820       is_extension__ = PETSC_FALSE; \
5821     } else { \
5822       PetscCall(PetscStrncmp(filename + len - ext_len, extension__, ext_len, &is_extension__)); \
5823     } \
5824   } while (0)
5825 
5826   CheckExtension(extGmsh, isGmsh);
5827   CheckExtension(extGmsh2, isGmsh2);
5828   CheckExtension(extGmsh4, isGmsh4);
5829   CheckExtension(extCGNS, isCGNS);
5830   CheckExtension(extExodus, isExodus);
5831   if (!isExodus) CheckExtension(extExodus_e, isExodus);
5832   CheckExtension(extGenesis, isGenesis);
5833   CheckExtension(extFluent, isFluent);
5834   CheckExtension(extHDF5, isHDF5);
5835   CheckExtension(extPLY, isPLY);
5836   CheckExtension(extEGADSLite, isEGADSLite);
5837   CheckExtension(extEGADS, isEGADS);
5838   CheckExtension(extIGES, isIGES);
5839   CheckExtension(extSTEP, isSTEP);
5840   CheckExtension(extCV, isCV);
5841   CheckExtension(extXDMFHDF5, isXDMFHDF5);
5842 
5843 #undef CheckExtension
5844 
5845   if (isGmsh || isGmsh2 || isGmsh4) {
5846     PetscCall(DMPlexCreateGmshFromFile(comm, filename, interpolate, dm));
5847   } else if (isCGNS) {
5848     PetscCall(DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm));
5849   } else if (isExodus || isGenesis) {
5850     PetscCall(DMPlexCreateExodusFromFile(comm, filename, interpolate, dm));
5851   } else if (isFluent) {
5852     PetscCall(DMPlexCreateFluentFromFile(comm, filename, interpolate, dm));
5853   } else if (isHDF5) {
5854     PetscViewer viewer;
5855 
5856     /* PETSC_VIEWER_HDF5_XDMF is used if the filename ends with .xdmf.h5, or if -dm_plex_create_from_hdf5_xdmf option is present */
5857     PetscCall(PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &isXDMFHDF5, NULL));
5858     PetscCall(PetscViewerCreate(comm, &viewer));
5859     PetscCall(PetscViewerSetType(viewer, PETSCVIEWERHDF5));
5860     PetscCall(PetscViewerSetOptionsPrefix(viewer, "dm_plex_create_"));
5861     PetscCall(PetscViewerSetFromOptions(viewer));
5862     PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
5863     PetscCall(PetscViewerFileSetName(viewer, filename));
5864 
5865     PetscCall(DMCreate(comm, dm));
5866     PetscCall(PetscObjectSetName((PetscObject)*dm, plexname));
5867     PetscCall(DMSetType(*dm, DMPLEX));
5868     if (isXDMFHDF5) PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF));
5869     PetscCall(DMLoad(*dm, viewer));
5870     if (isXDMFHDF5) PetscCall(PetscViewerPopFormat(viewer));
5871     PetscCall(PetscViewerDestroy(&viewer));
5872 
5873     if (interpolate) {
5874       DM idm;
5875 
5876       PetscCall(DMPlexInterpolate(*dm, &idm));
5877       PetscCall(DMDestroy(dm));
5878       *dm = idm;
5879     }
5880   } else if (isPLY) {
5881     PetscCall(DMPlexCreatePLYFromFile(comm, filename, interpolate, dm));
5882   } else if (isEGADSLite || isEGADS || isIGES || isSTEP) {
5883     if (isEGADSLite) PetscCall(DMPlexCreateEGADSLiteFromFile(comm, filename, dm));
5884     else PetscCall(DMPlexCreateEGADSFromFile(comm, filename, dm));
5885     if (!interpolate) {
5886       DM udm;
5887 
5888       PetscCall(DMPlexUninterpolate(*dm, &udm));
5889       PetscCall(DMDestroy(dm));
5890       *dm = udm;
5891     }
5892   } else if (isCV) {
5893     PetscCall(DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm));
5894   } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
5895   PetscCall(PetscStrlen(plexname, &len));
5896   if (len) PetscCall(PetscObjectSetName((PetscObject)*dm, plexname));
5897   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5898   PetscFunctionReturn(PETSC_SUCCESS);
5899 }
5900 
5901 /*@
5902   DMPlexCreateEphemeral - This takes a `DMPlexTransform` and a base `DMPlex` and produces an ephemeral `DM`, meaning one that is created on the fly in response to queries.
5903 
5904   Input Parameters:
5905 + tr     - The `DMPlexTransform`
5906 - prefix - An options prefix, or NULL
5907 
5908   Output Parameter:
5909 . dm - The `DM`
5910 
5911   Level: beginner
5912 
5913   Notes:
5914   An emphemeral mesh is one that is not stored concretely, as in the default `DMPLEX` implementation, but rather is produced on the fly in response to queries, using information from the transform and the base mesh.
5915 
5916 .seealso: `DMPlexCreateFromFile`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
5917 @*/
5918 PetscErrorCode DMPlexCreateEphemeral(DMPlexTransform tr, const char prefix[], DM *dm)
5919 {
5920   DM           bdm, bcdm, cdm;
5921   Vec          coordinates, coordinatesNew;
5922   PetscSection cs;
5923   PetscInt     dim, cdim, Nl;
5924 
5925   PetscFunctionBegin;
5926   PetscCall(DMCreate(PetscObjectComm((PetscObject)tr), dm));
5927   PetscCall(DMSetType(*dm, DMPLEX));
5928   ((DM_Plex *)(*dm)->data)->interpolated = DMPLEX_INTERPOLATED_FULL;
5929   // Handle coordinates
5930   PetscCall(DMPlexTransformGetDM(tr, &bdm));
5931   PetscCall(DMGetCoordinateDim(bdm, &cdim));
5932   PetscCall(DMSetCoordinateDim(*dm, cdim));
5933   PetscCall(DMGetDimension(bdm, &dim));
5934   PetscCall(DMSetDimension(*dm, dim));
5935   PetscCall(DMGetCoordinateDM(bdm, &bcdm));
5936   PetscCall(DMGetCoordinateDM(*dm, &cdm));
5937   PetscCall(DMCopyDisc(bcdm, cdm));
5938   PetscCall(DMGetLocalSection(cdm, &cs));
5939   PetscCall(PetscSectionSetNumFields(cs, 1));
5940   PetscCall(PetscSectionSetFieldComponents(cs, 0, cdim));
5941   PetscCall(DMGetCoordinatesLocal(bdm, &coordinates));
5942   PetscCall(VecDuplicate(coordinates, &coordinatesNew));
5943   PetscCall(VecCopy(coordinates, coordinatesNew));
5944   PetscCall(DMSetCoordinatesLocal(*dm, coordinatesNew));
5945   PetscCall(VecDestroy(&coordinatesNew));
5946 
5947   PetscCall(PetscObjectReference((PetscObject)tr));
5948   PetscCall(DMPlexTransformDestroy(&((DM_Plex *)(*dm)->data)->tr));
5949   ((DM_Plex *)(*dm)->data)->tr = tr;
5950   PetscCall(DMPlexDistributeSetDefault(*dm, PETSC_FALSE));
5951   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)*dm, prefix));
5952   PetscCall(DMSetFromOptions(*dm));
5953 
5954   PetscCall(DMGetNumLabels(bdm, &Nl));
5955   for (PetscInt l = 0; l < Nl; ++l) {
5956     DMLabel     label, labelNew;
5957     const char *lname;
5958     PetscBool   isDepth, isCellType;
5959 
5960     PetscCall(DMGetLabelName(bdm, l, &lname));
5961     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
5962     if (isDepth) continue;
5963     PetscCall(PetscStrcmp(lname, "celltype", &isCellType));
5964     if (isCellType) continue;
5965     PetscCall(DMCreateLabel(*dm, lname));
5966     PetscCall(DMGetLabel(bdm, lname, &label));
5967     PetscCall(DMGetLabel(*dm, lname, &labelNew));
5968     PetscCall(DMLabelSetType(labelNew, DMLABELEPHEMERAL));
5969     PetscCall(DMLabelEphemeralSetLabel(labelNew, label));
5970     PetscCall(DMLabelEphemeralSetTransform(labelNew, tr));
5971     PetscCall(DMLabelSetUp(labelNew));
5972   }
5973   PetscFunctionReturn(PETSC_SUCCESS);
5974 }
5975