xref: /petsc/src/dm/impls/plex/plexcreate.c (revision b9d635d7b5c3ee5434768b771e5fbf76581fe7bc)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h> /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>
4 #include <petscsf.h>
5 #include <petscdmplextransform.h> /*I   "petscdmplextransform.h"   I*/
6 #include <petscdmlabelephemeral.h>
7 #include <petsc/private/kernels/blockmatmult.h>
8 #include <petsc/private/kernels/blockinvert.h>
9 
10 #ifdef PETSC_HAVE_UNISTD_H
11   #include <unistd.h>
12 #endif
13 #include <errno.h>
14 
15 PetscLogEvent DMPLEX_CreateFromFile, DMPLEX_CreateFromOptions, DMPLEX_BuildFromCellList, DMPLEX_BuildCoordinatesFromCellList;
16 
17 /* External function declarations here */
18 static PetscErrorCode DMInitialize_Plex(DM dm);
19 
20 /* This copies internal things in the Plex structure that we generally want when making a new, related Plex */
21 PetscErrorCode DMPlexCopy_Internal(DM dmin, PetscBool copyPeriodicity, PetscBool copyOverlap, DM dmout)
22 {
23   const PetscReal     *maxCell, *Lstart, *L;
24   VecType              vecType;
25   MatType              matType;
26   PetscBool            dist, useCeed;
27   DMReorderDefaultFlag reorder;
28 
29   PetscFunctionBegin;
30   if (dmin == dmout) PetscFunctionReturn(PETSC_SUCCESS);
31   PetscCall(DMGetVecType(dmin, &vecType));
32   PetscCall(DMSetVecType(dmout, vecType));
33   PetscCall(DMGetMatType(dmin, &matType));
34   PetscCall(DMSetMatType(dmout, matType));
35   if (copyPeriodicity) {
36     PetscCall(DMGetPeriodicity(dmin, &maxCell, &Lstart, &L));
37     PetscCall(DMSetPeriodicity(dmout, maxCell, Lstart, L));
38     PetscCall(DMLocalizeCoordinates(dmout));
39   }
40   PetscCall(DMPlexDistributeGetDefault(dmin, &dist));
41   PetscCall(DMPlexDistributeSetDefault(dmout, dist));
42   PetscCall(DMPlexReorderGetDefault(dmin, &reorder));
43   PetscCall(DMPlexReorderSetDefault(dmout, reorder));
44   PetscCall(DMPlexGetUseCeed(dmin, &useCeed));
45   PetscCall(DMPlexSetUseCeed(dmout, useCeed));
46   ((DM_Plex *)dmout->data)->useHashLocation = ((DM_Plex *)dmin->data)->useHashLocation;
47   ((DM_Plex *)dmout->data)->printSetValues  = ((DM_Plex *)dmin->data)->printSetValues;
48   ((DM_Plex *)dmout->data)->printFEM        = ((DM_Plex *)dmin->data)->printFEM;
49   ((DM_Plex *)dmout->data)->printFVM        = ((DM_Plex *)dmin->data)->printFVM;
50   ((DM_Plex *)dmout->data)->printL2         = ((DM_Plex *)dmin->data)->printL2;
51   ((DM_Plex *)dmout->data)->printLocate     = ((DM_Plex *)dmin->data)->printLocate;
52   ((DM_Plex *)dmout->data)->printProject    = ((DM_Plex *)dmin->data)->printProject;
53   ((DM_Plex *)dmout->data)->printTol        = ((DM_Plex *)dmin->data)->printTol;
54   if (copyOverlap) PetscCall(DMPlexSetOverlap_Plex(dmout, dmin, 0));
55   PetscFunctionReturn(PETSC_SUCCESS);
56 }
57 
58 /* Replace dm with the contents of ndm, and then destroy ndm
59    - Share the DM_Plex structure
60    - Share the coordinates
61    - Share the SF
62 */
63 PetscErrorCode DMPlexReplace_Internal(DM dm, DM *ndm)
64 {
65   PetscSF          sf;
66   DM               dmNew = *ndm, coordDM, coarseDM;
67   Vec              coords;
68   const PetscReal *maxCell, *Lstart, *L;
69   PetscInt         dim, cdim;
70   PetscBool        use_natural;
71 
72   PetscFunctionBegin;
73   if (dm == dmNew) {
74     PetscCall(DMDestroy(ndm));
75     PetscFunctionReturn(PETSC_SUCCESS);
76   }
77   dm->setupcalled = dmNew->setupcalled;
78   if (!dm->hdr.name) {
79     const char *name;
80 
81     PetscCall(PetscObjectGetName((PetscObject)*ndm, &name));
82     PetscCall(PetscObjectSetName((PetscObject)dm, name));
83   }
84   PetscCall(DMGetDimension(dmNew, &dim));
85   PetscCall(DMSetDimension(dm, dim));
86   PetscCall(DMGetCoordinateDim(dmNew, &cdim));
87   PetscCall(DMSetCoordinateDim(dm, cdim));
88   PetscCall(DMGetPointSF(dmNew, &sf));
89   PetscCall(DMSetPointSF(dm, sf));
90   PetscCall(DMGetCoordinateDM(dmNew, &coordDM));
91   PetscCall(DMGetCoordinatesLocal(dmNew, &coords));
92   PetscCall(DMSetCoordinateDM(dm, coordDM));
93   PetscCall(DMSetCoordinatesLocal(dm, coords));
94   PetscCall(DMGetCellCoordinateDM(dmNew, &coordDM));
95   PetscCall(DMGetCellCoordinatesLocal(dmNew, &coords));
96   PetscCall(DMSetCellCoordinateDM(dm, coordDM));
97   PetscCall(DMSetCellCoordinatesLocal(dm, coords));
98   /* Do not want to create the coordinate field if it does not already exist, so do not call DMGetCoordinateField() */
99   PetscCall(DMFieldDestroy(&dm->coordinates[0].field));
100   dm->coordinates[0].field            = dmNew->coordinates[0].field;
101   ((DM_Plex *)dmNew->data)->coordFunc = ((DM_Plex *)dm->data)->coordFunc;
102   PetscCall(DMGetPeriodicity(dmNew, &maxCell, &Lstart, &L));
103   PetscCall(DMSetPeriodicity(dm, maxCell, Lstart, L));
104   PetscCall(DMGetNaturalSF(dmNew, &sf));
105   PetscCall(DMSetNaturalSF(dm, sf));
106   PetscCall(DMGetUseNatural(dmNew, &use_natural));
107   PetscCall(DMSetUseNatural(dm, use_natural));
108   PetscCall(DMDestroy_Plex(dm));
109   PetscCall(DMInitialize_Plex(dm));
110   dm->data = dmNew->data;
111   ((DM_Plex *)dmNew->data)->refct++;
112   {
113     PetscInt       num_face_sfs;
114     const PetscSF *sfs;
115     PetscCall(DMPlexGetIsoperiodicFaceSF(dm, &num_face_sfs, &sfs));
116     PetscCall(DMPlexSetIsoperiodicFaceSF(dm, num_face_sfs, (PetscSF *)sfs)); // for the compose function effect on dm
117   }
118   PetscCall(DMDestroyLabelLinkList_Internal(dm));
119   PetscCall(DMCopyLabels(dmNew, dm, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL));
120   PetscCall(DMGetCoarseDM(dmNew, &coarseDM));
121   PetscCall(DMSetCoarseDM(dm, coarseDM));
122   PetscCall(DMDestroy(ndm));
123   PetscFunctionReturn(PETSC_SUCCESS);
124 }
125 
126 /* Swap dm with the contents of dmNew
127    - Swap the DM_Plex structure
128    - Swap the coordinates
129    - Swap the point PetscSF
130 */
131 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
132 {
133   DM          coordDMA, coordDMB;
134   Vec         coordsA, coordsB;
135   PetscSF     sfA, sfB;
136   DMField     fieldTmp;
137   void       *tmp;
138   DMLabelLink listTmp;
139   DMLabel     depthTmp;
140   PetscInt    tmpI;
141 
142   PetscFunctionBegin;
143   if (dmA == dmB) PetscFunctionReturn(PETSC_SUCCESS);
144   PetscCall(DMGetPointSF(dmA, &sfA));
145   PetscCall(DMGetPointSF(dmB, &sfB));
146   PetscCall(PetscObjectReference((PetscObject)sfA));
147   PetscCall(DMSetPointSF(dmA, sfB));
148   PetscCall(DMSetPointSF(dmB, sfA));
149   PetscCall(PetscObjectDereference((PetscObject)sfA));
150 
151   PetscCall(DMGetCoordinateDM(dmA, &coordDMA));
152   PetscCall(DMGetCoordinateDM(dmB, &coordDMB));
153   PetscCall(PetscObjectReference((PetscObject)coordDMA));
154   PetscCall(DMSetCoordinateDM(dmA, coordDMB));
155   PetscCall(DMSetCoordinateDM(dmB, coordDMA));
156   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
157 
158   PetscCall(DMGetCoordinatesLocal(dmA, &coordsA));
159   PetscCall(DMGetCoordinatesLocal(dmB, &coordsB));
160   PetscCall(PetscObjectReference((PetscObject)coordsA));
161   PetscCall(DMSetCoordinatesLocal(dmA, coordsB));
162   PetscCall(DMSetCoordinatesLocal(dmB, coordsA));
163   PetscCall(PetscObjectDereference((PetscObject)coordsA));
164 
165   PetscCall(DMGetCellCoordinateDM(dmA, &coordDMA));
166   PetscCall(DMGetCellCoordinateDM(dmB, &coordDMB));
167   PetscCall(PetscObjectReference((PetscObject)coordDMA));
168   PetscCall(DMSetCellCoordinateDM(dmA, coordDMB));
169   PetscCall(DMSetCellCoordinateDM(dmB, coordDMA));
170   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
171 
172   PetscCall(DMGetCellCoordinatesLocal(dmA, &coordsA));
173   PetscCall(DMGetCellCoordinatesLocal(dmB, &coordsB));
174   PetscCall(PetscObjectReference((PetscObject)coordsA));
175   PetscCall(DMSetCellCoordinatesLocal(dmA, coordsB));
176   PetscCall(DMSetCellCoordinatesLocal(dmB, coordsA));
177   PetscCall(PetscObjectDereference((PetscObject)coordsA));
178 
179   fieldTmp                  = dmA->coordinates[0].field;
180   dmA->coordinates[0].field = dmB->coordinates[0].field;
181   dmB->coordinates[0].field = fieldTmp;
182   fieldTmp                  = dmA->coordinates[1].field;
183   dmA->coordinates[1].field = dmB->coordinates[1].field;
184   dmB->coordinates[1].field = fieldTmp;
185   tmp                       = dmA->data;
186   dmA->data                 = dmB->data;
187   dmB->data                 = tmp;
188   listTmp                   = dmA->labels;
189   dmA->labels               = dmB->labels;
190   dmB->labels               = listTmp;
191   depthTmp                  = dmA->depthLabel;
192   dmA->depthLabel           = dmB->depthLabel;
193   dmB->depthLabel           = depthTmp;
194   depthTmp                  = dmA->celltypeLabel;
195   dmA->celltypeLabel        = dmB->celltypeLabel;
196   dmB->celltypeLabel        = depthTmp;
197   tmpI                      = dmA->levelup;
198   dmA->levelup              = dmB->levelup;
199   dmB->levelup              = tmpI;
200   PetscFunctionReturn(PETSC_SUCCESS);
201 }
202 
203 PetscErrorCode DMPlexInterpolateInPlace_Internal(DM dm)
204 {
205   DM idm;
206 
207   PetscFunctionBegin;
208   PetscCall(DMPlexInterpolate(dm, &idm));
209   PetscCall(DMPlexCopyCoordinates(dm, idm));
210   PetscCall(DMPlexReplace_Internal(dm, &idm));
211   PetscFunctionReturn(PETSC_SUCCESS);
212 }
213 
214 /*@C
215   DMPlexCreateCoordinateSpace - Creates a finite element space for the coordinates
216 
217   Collective
218 
219   Input Parameters:
220 + dm        - The `DMPLEX`
221 . degree    - The degree of the finite element or `PETSC_DECIDE`
222 . project   - Flag to project current coordinates into the space
223 - coordFunc - An optional function to map new points from refinement to the surface
224 
225   Level: advanced
226 
227 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `PetscPointFunc`, `PetscFECreateLagrange()`, `DMGetCoordinateDM()`
228 @*/
229 PetscErrorCode DMPlexCreateCoordinateSpace(DM dm, PetscInt degree, PetscBool project, PetscPointFunc coordFunc)
230 {
231   DM_Plex *mesh = (DM_Plex *)dm->data;
232   PetscFE  fe   = NULL;
233   DM       cdm;
234   PetscInt dim, dE, qorder, height;
235 
236   PetscFunctionBegin;
237   PetscCall(DMGetDimension(dm, &dim));
238   PetscCall(DMGetCoordinateDim(dm, &dE));
239   qorder = degree;
240   PetscCall(DMGetCoordinateDM(dm, &cdm));
241   PetscObjectOptionsBegin((PetscObject)cdm);
242   PetscCall(PetscOptionsBoundedInt("-default_quadrature_order", "Quadrature order is one less than quadrature points per edge", "DMPlexCreateCoordinateSpace", qorder, &qorder, NULL, 0));
243   PetscOptionsEnd();
244   PetscCall(DMPlexGetVTKCellHeight(dm, &height));
245   if (degree >= 0) {
246     DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
247     PetscInt       cStart, cEnd, gct;
248 
249     PetscCall(DMPlexGetHeightStratum(dm, height, &cStart, &cEnd));
250     if (cEnd > cStart) PetscCall(DMPlexGetCellType(dm, cStart, &ct));
251     gct = (PetscInt)ct;
252     PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &gct, 1, MPIU_INT, MPI_MIN, PetscObjectComm((PetscObject)dm)));
253     ct = (DMPolytopeType)gct;
254     // Work around current bug in PetscDualSpaceSetUp_Lagrange()
255     //   Can be seen in plex_tutorials-ex10_1
256     if (ct != DM_POLYTOPE_SEG_PRISM_TENSOR && ct != DM_POLYTOPE_TRI_PRISM_TENSOR && ct != DM_POLYTOPE_QUAD_PRISM_TENSOR) PetscCall(PetscFECreateLagrangeByCell(PETSC_COMM_SELF, dim, dE, ct, degree, qorder, &fe));
257   }
258   PetscCall(DMSetCoordinateDisc(dm, fe, project));
259   PetscCall(PetscFEDestroy(&fe));
260   mesh->coordFunc = coordFunc;
261   PetscFunctionReturn(PETSC_SUCCESS);
262 }
263 
264 /*@
265   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
266 
267   Collective
268 
269   Input Parameters:
270 + comm            - The communicator for the `DM` object
271 . dim             - The spatial dimension
272 . simplex         - Flag for simplicial cells, otherwise they are tensor product cells
273 . interpolate     - Flag to create intermediate mesh pieces (edges, faces)
274 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
275 
276   Output Parameter:
277 . newdm - The `DM` object
278 
279   Level: beginner
280 
281 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetType()`, `DMCreate()`
282 @*/
283 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscReal refinementLimit, DM *newdm)
284 {
285   DM          dm;
286   PetscMPIInt rank;
287 
288   PetscFunctionBegin;
289   PetscCall(DMCreate(comm, &dm));
290   PetscCall(DMSetType(dm, DMPLEX));
291   PetscCall(DMSetDimension(dm, dim));
292   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
293   PetscCallMPI(MPI_Comm_rank(comm, &rank));
294   switch (dim) {
295   case 2:
296     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "triangular"));
297     else PetscCall(PetscObjectSetName((PetscObject)dm, "quadrilateral"));
298     break;
299   case 3:
300     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "tetrahedral"));
301     else PetscCall(PetscObjectSetName((PetscObject)dm, "hexahedral"));
302     break;
303   default:
304     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
305   }
306   if (rank) {
307     PetscInt numPoints[2] = {0, 0};
308     PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL));
309   } else {
310     switch (dim) {
311     case 2:
312       if (simplex) {
313         PetscInt    numPoints[2]        = {4, 2};
314         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
315         PetscInt    cones[6]            = {2, 3, 4, 5, 4, 3};
316         PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
317         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
318 
319         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
320       } else {
321         PetscInt    numPoints[2]        = {6, 2};
322         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
323         PetscInt    cones[8]            = {2, 3, 4, 5, 3, 6, 7, 4};
324         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
325         PetscScalar vertexCoords[12]    = {-1.0, -0.5, 0.0, -0.5, 0.0, 0.5, -1.0, 0.5, 1.0, -0.5, 1.0, 0.5};
326 
327         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
328       }
329       break;
330     case 3:
331       if (simplex) {
332         PetscInt    numPoints[2]        = {5, 2};
333         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
334         PetscInt    cones[8]            = {4, 3, 5, 2, 5, 3, 4, 6};
335         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
336         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0};
337 
338         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
339       } else {
340         PetscInt    numPoints[2]         = {12, 2};
341         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
342         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9, 5, 4, 10, 11, 7, 12, 13, 8};
343         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
344         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5, -1.0, 0.5, -0.5, 0.0, 0.5, -0.5, 0.0, -0.5, -0.5, -1.0, -0.5, 0.5, 0.0, -0.5, 0.5, 0.0, 0.5, 0.5, -1.0, 0.5, 0.5, 1.0, 0.5, -0.5, 1.0, -0.5, -0.5, 1.0, -0.5, 0.5, 1.0, 0.5, 0.5};
345 
346         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
347       }
348       break;
349     default:
350       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
351     }
352   }
353   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
354   *newdm = dm;
355   if (refinementLimit > 0.0) {
356     DM          rdm;
357     const char *name;
358 
359     PetscCall(DMPlexSetRefinementUniform(*newdm, PETSC_FALSE));
360     PetscCall(DMPlexSetRefinementLimit(*newdm, refinementLimit));
361     PetscCall(DMRefine(*newdm, comm, &rdm));
362     PetscCall(PetscObjectGetName((PetscObject)*newdm, &name));
363     PetscCall(PetscObjectSetName((PetscObject)rdm, name));
364     PetscCall(DMDestroy(newdm));
365     *newdm = rdm;
366   }
367   if (interpolate) {
368     DM idm;
369 
370     PetscCall(DMPlexInterpolate(*newdm, &idm));
371     PetscCall(DMDestroy(newdm));
372     *newdm = idm;
373   }
374   PetscFunctionReturn(PETSC_SUCCESS);
375 }
376 
377 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
378 {
379   const PetscInt numVertices    = 2;
380   PetscInt       markerRight    = 1;
381   PetscInt       markerLeft     = 1;
382   PetscBool      markerSeparate = PETSC_FALSE;
383   Vec            coordinates;
384   PetscSection   coordSection;
385   PetscScalar   *coords;
386   PetscInt       coordSize;
387   PetscMPIInt    rank;
388   PetscInt       cdim = 1, v;
389 
390   PetscFunctionBegin;
391   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
392   if (markerSeparate) {
393     markerRight = 2;
394     markerLeft  = 1;
395   }
396   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
397   if (rank == 0) {
398     PetscCall(DMPlexSetChart(dm, 0, numVertices));
399     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
400     PetscCall(DMSetLabelValue(dm, "marker", 0, markerLeft));
401     PetscCall(DMSetLabelValue(dm, "marker", 1, markerRight));
402   }
403   PetscCall(DMPlexSymmetrize(dm));
404   PetscCall(DMPlexStratify(dm));
405   /* Build coordinates */
406   PetscCall(DMSetCoordinateDim(dm, cdim));
407   PetscCall(DMGetCoordinateSection(dm, &coordSection));
408   PetscCall(PetscSectionSetNumFields(coordSection, 1));
409   PetscCall(PetscSectionSetChart(coordSection, 0, numVertices));
410   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
411   for (v = 0; v < numVertices; ++v) {
412     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
413     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
414   }
415   PetscCall(PetscSectionSetUp(coordSection));
416   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
417   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
418   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
419   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
420   PetscCall(VecSetBlockSize(coordinates, cdim));
421   PetscCall(VecSetType(coordinates, VECSTANDARD));
422   PetscCall(VecGetArray(coordinates, &coords));
423   coords[0] = lower[0];
424   coords[1] = upper[0];
425   PetscCall(VecRestoreArray(coordinates, &coords));
426   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
427   PetscCall(VecDestroy(&coordinates));
428   PetscFunctionReturn(PETSC_SUCCESS);
429 }
430 
431 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
432 {
433   const PetscInt numVertices    = (edges[0] + 1) * (edges[1] + 1);
434   const PetscInt numEdges       = edges[0] * (edges[1] + 1) + (edges[0] + 1) * edges[1];
435   PetscInt       markerTop      = 1;
436   PetscInt       markerBottom   = 1;
437   PetscInt       markerRight    = 1;
438   PetscInt       markerLeft     = 1;
439   PetscBool      markerSeparate = PETSC_FALSE;
440   Vec            coordinates;
441   PetscSection   coordSection;
442   PetscScalar   *coords;
443   PetscInt       coordSize;
444   PetscMPIInt    rank;
445   PetscInt       v, vx, vy;
446 
447   PetscFunctionBegin;
448   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
449   if (markerSeparate) {
450     markerTop    = 3;
451     markerBottom = 1;
452     markerRight  = 2;
453     markerLeft   = 4;
454   }
455   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
456   if (rank == 0) {
457     PetscInt e, ex, ey;
458 
459     PetscCall(DMPlexSetChart(dm, 0, numEdges + numVertices));
460     for (e = 0; e < numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
461     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
462     for (vx = 0; vx <= edges[0]; vx++) {
463       for (ey = 0; ey < edges[1]; ey++) {
464         PetscInt edge   = vx * edges[1] + ey + edges[0] * (edges[1] + 1);
465         PetscInt vertex = ey * (edges[0] + 1) + vx + numEdges;
466         PetscInt cone[2];
467 
468         cone[0] = vertex;
469         cone[1] = vertex + edges[0] + 1;
470         PetscCall(DMPlexSetCone(dm, edge, cone));
471         if (vx == edges[0]) {
472           PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
473           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
474           if (ey == edges[1] - 1) {
475             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
476             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerRight));
477           }
478         } else if (vx == 0) {
479           PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
480           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
481           if (ey == edges[1] - 1) {
482             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
483             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft));
484           }
485         }
486       }
487     }
488     for (vy = 0; vy <= edges[1]; vy++) {
489       for (ex = 0; ex < edges[0]; ex++) {
490         PetscInt edge   = vy * edges[0] + ex;
491         PetscInt vertex = vy * (edges[0] + 1) + ex + numEdges;
492         PetscInt cone[2];
493 
494         cone[0] = vertex;
495         cone[1] = vertex + 1;
496         PetscCall(DMPlexSetCone(dm, edge, cone));
497         if (vy == edges[1]) {
498           PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
499           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
500           if (ex == edges[0] - 1) {
501             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
502             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerTop));
503           }
504         } else if (vy == 0) {
505           PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
506           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
507           if (ex == edges[0] - 1) {
508             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
509             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom));
510           }
511         }
512       }
513     }
514   }
515   PetscCall(DMPlexSymmetrize(dm));
516   PetscCall(DMPlexStratify(dm));
517   /* Build coordinates */
518   PetscCall(DMSetCoordinateDim(dm, 2));
519   PetscCall(DMGetCoordinateSection(dm, &coordSection));
520   PetscCall(PetscSectionSetNumFields(coordSection, 1));
521   PetscCall(PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices));
522   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 2));
523   for (v = numEdges; v < numEdges + numVertices; ++v) {
524     PetscCall(PetscSectionSetDof(coordSection, v, 2));
525     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 2));
526   }
527   PetscCall(PetscSectionSetUp(coordSection));
528   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
529   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
530   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
531   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
532   PetscCall(VecSetBlockSize(coordinates, 2));
533   PetscCall(VecSetType(coordinates, VECSTANDARD));
534   PetscCall(VecGetArray(coordinates, &coords));
535   for (vy = 0; vy <= edges[1]; ++vy) {
536     for (vx = 0; vx <= edges[0]; ++vx) {
537       coords[(vy * (edges[0] + 1) + vx) * 2 + 0] = lower[0] + ((upper[0] - lower[0]) / edges[0]) * vx;
538       coords[(vy * (edges[0] + 1) + vx) * 2 + 1] = lower[1] + ((upper[1] - lower[1]) / edges[1]) * vy;
539     }
540   }
541   PetscCall(VecRestoreArray(coordinates, &coords));
542   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
543   PetscCall(VecDestroy(&coordinates));
544   PetscFunctionReturn(PETSC_SUCCESS);
545 }
546 
547 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
548 {
549   PetscInt     vertices[3], numVertices;
550   PetscInt     numFaces       = 2 * faces[0] * faces[1] + 2 * faces[1] * faces[2] + 2 * faces[0] * faces[2];
551   PetscInt     markerTop      = 1;
552   PetscInt     markerBottom   = 1;
553   PetscInt     markerFront    = 1;
554   PetscInt     markerBack     = 1;
555   PetscInt     markerRight    = 1;
556   PetscInt     markerLeft     = 1;
557   PetscBool    markerSeparate = PETSC_FALSE;
558   Vec          coordinates;
559   PetscSection coordSection;
560   PetscScalar *coords;
561   PetscInt     coordSize;
562   PetscMPIInt  rank;
563   PetscInt     v, vx, vy, vz;
564   PetscInt     voffset, iface = 0, cone[4];
565 
566   PetscFunctionBegin;
567   PetscCheck(faces[0] >= 1 && faces[1] >= 1 && faces[2] >= 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
568   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
569   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
570   if (markerSeparate) {
571     markerBottom = 1;
572     markerTop    = 2;
573     markerFront  = 3;
574     markerBack   = 4;
575     markerRight  = 5;
576     markerLeft   = 6;
577   }
578   vertices[0] = faces[0] + 1;
579   vertices[1] = faces[1] + 1;
580   vertices[2] = faces[2] + 1;
581   numVertices = vertices[0] * vertices[1] * vertices[2];
582   if (rank == 0) {
583     PetscInt f;
584 
585     PetscCall(DMPlexSetChart(dm, 0, numFaces + numVertices));
586     for (f = 0; f < numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
587     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
588 
589     /* Side 0 (Top) */
590     for (vy = 0; vy < faces[1]; vy++) {
591       for (vx = 0; vx < faces[0]; vx++) {
592         voffset = numFaces + vertices[0] * vertices[1] * (vertices[2] - 1) + vy * vertices[0] + vx;
593         cone[0] = voffset;
594         cone[1] = voffset + 1;
595         cone[2] = voffset + vertices[0] + 1;
596         cone[3] = voffset + vertices[0];
597         PetscCall(DMPlexSetCone(dm, iface, cone));
598         PetscCall(DMSetLabelValue(dm, "marker", iface, markerTop));
599         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerTop));
600         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerTop));
601         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerTop));
602         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerTop));
603         iface++;
604       }
605     }
606 
607     /* Side 1 (Bottom) */
608     for (vy = 0; vy < faces[1]; vy++) {
609       for (vx = 0; vx < faces[0]; vx++) {
610         voffset = numFaces + vy * (faces[0] + 1) + vx;
611         cone[0] = voffset + 1;
612         cone[1] = voffset;
613         cone[2] = voffset + vertices[0];
614         cone[3] = voffset + vertices[0] + 1;
615         PetscCall(DMPlexSetCone(dm, iface, cone));
616         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBottom));
617         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBottom));
618         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBottom));
619         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerBottom));
620         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerBottom));
621         iface++;
622       }
623     }
624 
625     /* Side 2 (Front) */
626     for (vz = 0; vz < faces[2]; vz++) {
627       for (vx = 0; vx < faces[0]; vx++) {
628         voffset = numFaces + vz * vertices[0] * vertices[1] + vx;
629         cone[0] = voffset;
630         cone[1] = voffset + 1;
631         cone[2] = voffset + vertices[0] * vertices[1] + 1;
632         cone[3] = voffset + vertices[0] * vertices[1];
633         PetscCall(DMPlexSetCone(dm, iface, cone));
634         PetscCall(DMSetLabelValue(dm, "marker", iface, markerFront));
635         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerFront));
636         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerFront));
637         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerFront));
638         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerFront));
639         iface++;
640       }
641     }
642 
643     /* Side 3 (Back) */
644     for (vz = 0; vz < faces[2]; vz++) {
645       for (vx = 0; vx < faces[0]; vx++) {
646         voffset = numFaces + vz * vertices[0] * vertices[1] + vertices[0] * (vertices[1] - 1) + vx;
647         cone[0] = voffset + vertices[0] * vertices[1];
648         cone[1] = voffset + vertices[0] * vertices[1] + 1;
649         cone[2] = voffset + 1;
650         cone[3] = voffset;
651         PetscCall(DMPlexSetCone(dm, iface, cone));
652         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBack));
653         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBack));
654         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBack));
655         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerBack));
656         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerBack));
657         iface++;
658       }
659     }
660 
661     /* Side 4 (Left) */
662     for (vz = 0; vz < faces[2]; vz++) {
663       for (vy = 0; vy < faces[1]; vy++) {
664         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0];
665         cone[0] = voffset;
666         cone[1] = voffset + vertices[0] * vertices[1];
667         cone[2] = voffset + vertices[0] * vertices[1] + vertices[0];
668         cone[3] = voffset + vertices[0];
669         PetscCall(DMPlexSetCone(dm, iface, cone));
670         PetscCall(DMSetLabelValue(dm, "marker", iface, markerLeft));
671         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerLeft));
672         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerLeft));
673         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[1] + 0, markerLeft));
674         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerLeft));
675         iface++;
676       }
677     }
678 
679     /* Side 5 (Right) */
680     for (vz = 0; vz < faces[2]; vz++) {
681       for (vy = 0; vy < faces[1]; vy++) {
682         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0] + faces[0];
683         cone[0] = voffset + vertices[0] * vertices[1];
684         cone[1] = voffset;
685         cone[2] = voffset + vertices[0];
686         cone[3] = voffset + vertices[0] * vertices[1] + vertices[0];
687         PetscCall(DMPlexSetCone(dm, iface, cone));
688         PetscCall(DMSetLabelValue(dm, "marker", iface, markerRight));
689         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerRight));
690         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerRight));
691         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerRight));
692         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerRight));
693         iface++;
694       }
695     }
696   }
697   PetscCall(DMPlexSymmetrize(dm));
698   PetscCall(DMPlexStratify(dm));
699   /* Build coordinates */
700   PetscCall(DMSetCoordinateDim(dm, 3));
701   PetscCall(DMGetCoordinateSection(dm, &coordSection));
702   PetscCall(PetscSectionSetNumFields(coordSection, 1));
703   PetscCall(PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices));
704   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 3));
705   for (v = numFaces; v < numFaces + numVertices; ++v) {
706     PetscCall(PetscSectionSetDof(coordSection, v, 3));
707     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 3));
708   }
709   PetscCall(PetscSectionSetUp(coordSection));
710   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
711   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
712   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
713   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
714   PetscCall(VecSetBlockSize(coordinates, 3));
715   PetscCall(VecSetType(coordinates, VECSTANDARD));
716   PetscCall(VecGetArray(coordinates, &coords));
717   for (vz = 0; vz <= faces[2]; ++vz) {
718     for (vy = 0; vy <= faces[1]; ++vy) {
719       for (vx = 0; vx <= faces[0]; ++vx) {
720         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 0] = lower[0] + ((upper[0] - lower[0]) / faces[0]) * vx;
721         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 1] = lower[1] + ((upper[1] - lower[1]) / faces[1]) * vy;
722         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 2] = lower[2] + ((upper[2] - lower[2]) / faces[2]) * vz;
723       }
724     }
725   }
726   PetscCall(VecRestoreArray(coordinates, &coords));
727   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
728   PetscCall(VecDestroy(&coordinates));
729   PetscFunctionReturn(PETSC_SUCCESS);
730 }
731 
732 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate)
733 {
734   PetscFunctionBegin;
735   PetscValidLogicalCollectiveInt(dm, dim, 2);
736   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
737   PetscCall(DMSetDimension(dm, dim - 1));
738   PetscCall(DMSetCoordinateDim(dm, dim));
739   switch (dim) {
740   case 1:
741     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(dm, lower, upper, faces));
742     break;
743   case 2:
744     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(dm, lower, upper, faces));
745     break;
746   case 3:
747     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(dm, lower, upper, faces));
748     break;
749   default:
750     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Dimension not supported: %" PetscInt_FMT, dim);
751   }
752   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
753   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
754   PetscFunctionReturn(PETSC_SUCCESS);
755 }
756 
757 /*@C
758   DMPlexCreateBoxSurfaceMesh - Creates a mesh on the surface of the tensor product of unit intervals (box) using tensor cells (hexahedra).
759 
760   Collective
761 
762   Input Parameters:
763 + comm        - The communicator for the `DM` object
764 . dim         - The spatial dimension of the box, so the resulting mesh is has dimension `dim`-1
765 . faces       - Number of faces per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
766 . lower       - The lower left corner, or `NULL` for (0, 0, 0)
767 . upper       - The upper right corner, or `NULL` for (1, 1, 1)
768 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
769 
770   Output Parameter:
771 . dm - The `DM` object
772 
773   Level: beginner
774 
775 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateBoxMesh()`, `DMPlexCreateFromFile()`, `DMSetType()`, `DMCreate()`
776 @*/
777 PetscErrorCode DMPlexCreateBoxSurfaceMesh(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate, DM *dm)
778 {
779   PetscInt  fac[3] = {1, 1, 1};
780   PetscReal low[3] = {0, 0, 0};
781   PetscReal upp[3] = {1, 1, 1};
782 
783   PetscFunctionBegin;
784   PetscCall(DMCreate(comm, dm));
785   PetscCall(DMSetType(*dm, DMPLEX));
786   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(*dm, dim, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, interpolate));
787   PetscFunctionReturn(PETSC_SUCCESS);
788 }
789 
790 static PetscErrorCode DMPlexCreateLineMesh_Internal(DM dm, PetscInt segments, PetscReal lower, PetscReal upper, DMBoundaryType bd)
791 {
792   PetscInt     i, fStart, fEnd, numCells = 0, numVerts = 0;
793   PetscInt     numPoints[2], *coneSize, *cones, *coneOrientations;
794   PetscScalar *vertexCoords;
795   PetscReal    L, maxCell;
796   PetscBool    markerSeparate = PETSC_FALSE;
797   PetscInt     markerLeft = 1, faceMarkerLeft = 1;
798   PetscInt     markerRight = 1, faceMarkerRight = 2;
799   PetscBool    wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
800   PetscMPIInt  rank;
801 
802   PetscFunctionBegin;
803   PetscAssertPointer(dm, 1);
804 
805   PetscCall(DMSetDimension(dm, 1));
806   PetscCall(DMCreateLabel(dm, "marker"));
807   PetscCall(DMCreateLabel(dm, "Face Sets"));
808 
809   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
810   if (rank == 0) numCells = segments;
811   if (rank == 0) numVerts = segments + (wrap ? 0 : 1);
812 
813   numPoints[0] = numVerts;
814   numPoints[1] = numCells;
815   PetscCall(PetscMalloc4(numCells + numVerts, &coneSize, numCells * 2, &cones, numCells + numVerts, &coneOrientations, numVerts, &vertexCoords));
816   PetscCall(PetscArrayzero(coneOrientations, numCells + numVerts));
817   for (i = 0; i < numCells; ++i) coneSize[i] = 2;
818   for (i = 0; i < numVerts; ++i) coneSize[numCells + i] = 0;
819   for (i = 0; i < numCells; ++i) {
820     cones[2 * i]     = numCells + i % numVerts;
821     cones[2 * i + 1] = numCells + (i + 1) % numVerts;
822   }
823   for (i = 0; i < numVerts; ++i) vertexCoords[i] = lower + (upper - lower) * ((PetscReal)i / (PetscReal)numCells);
824   PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
825   PetscCall(PetscFree4(coneSize, cones, coneOrientations, vertexCoords));
826 
827   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
828   if (markerSeparate) {
829     markerLeft  = faceMarkerLeft;
830     markerRight = faceMarkerRight;
831   }
832   if (!wrap && rank == 0) {
833     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
834     PetscCall(DMSetLabelValue(dm, "marker", fStart, markerLeft));
835     PetscCall(DMSetLabelValue(dm, "marker", fEnd - 1, markerRight));
836     PetscCall(DMSetLabelValue(dm, "Face Sets", fStart, faceMarkerLeft));
837     PetscCall(DMSetLabelValue(dm, "Face Sets", fEnd - 1, faceMarkerRight));
838   }
839   if (wrap) {
840     L       = upper - lower;
841     maxCell = (PetscReal)1.1 * (L / (PetscReal)PetscMax(1, segments));
842     PetscCall(DMSetPeriodicity(dm, &maxCell, &lower, &L));
843   }
844   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
845   PetscFunctionReturn(PETSC_SUCCESS);
846 }
847 
848 // Creates "Face Sets" label based on the standard box labeling conventions
849 static PetscErrorCode DMPlexSetBoxLabel_Internal(DM dm, const DMBoundaryType periodicity[])
850 {
851   DM              cdm;
852   PetscSection    csection;
853   Vec             coordinates;
854   DMLabel         label;
855   IS              faces_is;
856   PetscInt        dim, num_face = 0;
857   const PetscInt *faces;
858   PetscInt        faceMarkerBottom, faceMarkerTop, faceMarkerFront, faceMarkerBack, faceMarkerRight, faceMarkerLeft;
859 
860   PetscFunctionBeginUser;
861   PetscCall(DMGetDimension(dm, &dim));
862   PetscCheck((dim == 2) || (dim == 3), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "DMPlex box labeling only supports 2D and 3D meshes, received DM of dimension %" PetscInt_FMT, dim);
863   // Get Face Sets label
864   PetscCall(DMGetLabel(dm, "Face Sets", &label));
865   if (label) {
866     PetscCall(DMLabelReset(label));
867   } else {
868     PetscCall(DMCreateLabel(dm, "Face Sets"));
869     PetscCall(DMGetLabel(dm, "Face Sets", &label));
870   }
871   PetscCall(DMPlexMarkBoundaryFaces(dm, 1, label));
872   PetscCall(DMGetStratumIS(dm, "Face Sets", 1, &faces_is));
873 
874   switch (dim) {
875   case 2:
876     faceMarkerTop    = 3;
877     faceMarkerBottom = 1;
878     faceMarkerRight  = 2;
879     faceMarkerLeft   = 4;
880     break;
881   case 3:
882     faceMarkerBottom = 1;
883     faceMarkerTop    = 2;
884     faceMarkerFront  = 3;
885     faceMarkerBack   = 4;
886     faceMarkerRight  = 5;
887     faceMarkerLeft   = 6;
888     break;
889   default:
890     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
891   }
892 
893   if (faces_is) PetscCall(ISGetLocalSize(faces_is, &num_face));
894   if (faces_is) PetscCall(ISGetIndices(faces_is, &faces));
895   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
896   PetscCall(DMGetCoordinateDM(dm, &cdm));
897   PetscCall(DMGetLocalSection(cdm, &csection));
898   for (PetscInt f = 0; f < num_face; ++f) {
899     PetscScalar *coords = NULL;
900     PetscInt     face = faces[f], flip = 1, label_value = -1, coords_size;
901 
902     { // Determine if orientation of face is flipped
903       PetscInt        num_cells_support, num_faces, start = -1;
904       const PetscInt *orients, *cell_faces, *cells;
905 
906       PetscCall(DMPlexGetSupport(dm, face, &cells));
907       PetscCall(DMPlexGetSupportSize(dm, face, &num_cells_support));
908       PetscCheck(num_cells_support == 1, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Expected one cell in support of exterior face, but got %" PetscInt_FMT " cells", num_cells_support);
909       PetscCall(DMPlexGetCone(dm, cells[0], &cell_faces));
910       PetscCall(DMPlexGetConeSize(dm, cells[0], &num_faces));
911       for (PetscInt i = 0; i < num_faces; i++) {
912         if (cell_faces[i] == face) start = i;
913       }
914       PetscCheck(start >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_CORRUPT, "Could not find face %" PetscInt_FMT " in cone of its support", face);
915       PetscCall(DMPlexGetConeOrientation(dm, cells[0], &orients));
916       if (orients[start] < 0) flip = -1;
917     }
918 
919     // Cannot use DMPlexComputeCellGeometryFVM() for high-order geometry, so must calculate normal vectors manually
920     // Use the vertices (depth 0) of coordinate DM to calculate normal vector
921     PetscCall(DMPlexVecGetClosureAtDepth_Internal(cdm, csection, coordinates, face, 0, &coords_size, &coords));
922     switch (dim) {
923     case 2: {
924       PetscScalar vec[2];
925 
926       for (PetscInt d = 0; d < dim; ++d) vec[d] = flip * (PetscRealPart(coords[1 * dim + d]) - PetscRealPart(coords[0 * dim + d]));
927       PetscScalar normal[] = {vec[1], -vec[0]};
928       if (PetscAbsScalar(normal[0]) > PetscAbsScalar(normal[1])) {
929         label_value = PetscRealPart(normal[0]) > 0 ? faceMarkerRight : faceMarkerLeft;
930       } else {
931         label_value = PetscRealPart(normal[1]) > 0 ? faceMarkerTop : faceMarkerBottom;
932       }
933     } break;
934     case 3: {
935       PetscScalar vec1[3], vec2[3], normal[3];
936 
937       for (PetscInt d = 0; d < dim; ++d) {
938         vec1[d] = PetscRealPart(coords[1 * dim + d]) - PetscRealPart(coords[0 * dim + d]);
939         vec2[d] = PetscRealPart(coords[2 * dim + d]) - PetscRealPart(coords[1 * dim + d]);
940       }
941 
942       // Calculate normal vector via cross-product
943       normal[0] = flip * ((vec1[1] * vec2[2]) - (vec1[2] * vec2[1]));
944       normal[1] = flip * ((vec1[2] * vec2[0]) - (vec1[0] * vec2[2]));
945       normal[2] = flip * ((vec1[0] * vec2[1]) - (vec1[1] * vec2[0]));
946 
947       if (PetscAbsScalar(normal[0]) > PetscAbsScalar(normal[1])) {
948         if (PetscAbsScalar(normal[0]) > PetscAbsScalar(normal[2])) {
949           label_value = PetscRealPart(normal[0]) > 0 ? faceMarkerRight : faceMarkerLeft;
950         } else {
951           label_value = PetscRealPart(normal[2]) > 0 ? faceMarkerTop : faceMarkerBottom;
952         }
953       } else {
954         if (PetscAbsScalar(normal[1]) > PetscAbsScalar(normal[2])) {
955           label_value = PetscRealPart(normal[1]) > 0 ? faceMarkerBack : faceMarkerFront;
956         } else {
957           label_value = PetscRealPart(normal[2]) > 0 ? faceMarkerTop : faceMarkerBottom;
958         }
959       }
960     } break;
961     }
962 
963     PetscInt previous_label_value; // always 1 due to DMPlexMarkBoundaryFaces call above
964     PetscCall(DMGetLabelValue(dm, "Face Sets", face, &previous_label_value));
965     PetscCall(DMClearLabelValue(dm, "Face Sets", face, previous_label_value));
966     PetscCall(DMSetLabelValue(dm, "Face Sets", face, label_value));
967     PetscCall(DMPlexVecRestoreClosure(cdm, csection, coordinates, face, &coords_size, &coords));
968   }
969   if (faces_is) PetscCall(ISRestoreIndices(faces_is, &faces));
970   PetscCall(ISDestroy(&faces_is));
971 
972   // Create Isoperiodic SF from newly-created face labels
973   PetscSF     periodicsfs[3];
974   PetscInt    periodic_sf_index  = 0;
975   PetscScalar transform[3][4][4] = {{{0.}}};
976   for (PetscInt d = 0; d < dim; d++) {
977     IS              donor_is, periodic_is;
978     const PetscInt *donor_faces = NULL, *periodic_faces = NULL;
979     PetscInt        num_donor = 0, num_periodic = 0;
980     PetscSF         centroidsf;
981     PetscReal       donor_to_periodic_distance;
982     const PetscInt  face_pairings[2][3][2] = {
983       // 2D face pairings, {donor, periodic}
984       {{4, 2}, {1, 3}},
985       // 3D face pairings
986       {{5, 6}, {3, 4}, {1, 2}}
987     };
988 
989     if (periodicity[d] != DM_BOUNDARY_PERIODIC) continue;
990     {
991       // Compute centroidsf, which is the mapping from donor faces to periodic faces
992       // Matches the centroid of the faces together, ignoring the periodic direction component (which should not match between donor and periodic face)
993       PetscInt     coords_size, centroid_comps = dim - 1;
994       PetscScalar *coords = NULL;
995       PetscReal   *donor_centroids, *periodic_centroids;
996       PetscReal    loc_periodic[2] = {PETSC_MIN_REAL, PETSC_MIN_REAL}, loc_periodic_global[2]; // Location of donor (0) and periodic (1) faces in periodic direction
997 
998       PetscCall(DMGetStratumIS(dm, "Face Sets", face_pairings[dim - 2][d][0], &donor_is));
999       PetscCall(DMGetStratumIS(dm, "Face Sets", face_pairings[dim - 2][d][1], &periodic_is));
1000       if (donor_is) {
1001         PetscCall(ISGetLocalSize(donor_is, &num_donor));
1002         PetscCall(ISGetIndices(donor_is, &donor_faces));
1003       }
1004       if (periodic_is) {
1005         PetscCall(ISGetLocalSize(periodic_is, &num_periodic));
1006         PetscCall(ISGetIndices(periodic_is, &periodic_faces));
1007       }
1008       PetscCall(PetscCalloc2(num_donor * centroid_comps, &donor_centroids, num_periodic * centroid_comps, &periodic_centroids));
1009       for (PetscInt f = 0; f < num_donor; f++) {
1010         PetscInt face = donor_faces[f], num_coords;
1011         PetscCall(DMPlexVecGetClosureAtDepth_Internal(cdm, csection, coordinates, face, 0, &coords_size, &coords));
1012         num_coords = coords_size / dim;
1013         for (PetscInt c = 0; c < num_coords; c++) {
1014           PetscInt comp_index = 0;
1015           loc_periodic[0]     = PetscRealPart(coords[c * dim + d]);
1016           for (PetscInt i = 0; i < dim; i++) {
1017             if (i == d) continue; // Periodic direction not used for centroid calculation
1018             donor_centroids[f * centroid_comps + comp_index] += PetscRealPart(coords[c * dim + i]) / num_coords;
1019             comp_index++;
1020           }
1021         }
1022         PetscCall(DMPlexVecRestoreClosure(cdm, csection, coordinates, face, &coords_size, &coords));
1023       }
1024 
1025       for (PetscInt f = 0; f < num_periodic; f++) {
1026         PetscInt face = periodic_faces[f], num_coords;
1027         PetscCall(DMPlexVecGetClosureAtDepth_Internal(cdm, csection, coordinates, face, 0, &coords_size, &coords));
1028         num_coords = coords_size / dim;
1029         for (PetscInt c = 0; c < num_coords; c++) {
1030           PetscInt comp_index = 0;
1031           loc_periodic[1]     = PetscRealPart(coords[c * dim + d]);
1032           for (PetscInt i = 0; i < dim; i++) {
1033             if (i == d) continue; // Periodic direction not used for centroid calculation
1034             periodic_centroids[f * centroid_comps + comp_index] += PetscRealPart(coords[c * dim + i]) / num_coords;
1035             comp_index++;
1036           }
1037         }
1038         PetscCall(DMPlexVecRestoreClosure(cdm, csection, coordinates, face, &coords_size, &coords));
1039       }
1040       PetscCallMPI(MPIU_Allreduce(loc_periodic, loc_periodic_global, 2, MPIU_REAL, MPIU_MAX, PetscObjectComm((PetscObject)dm)));
1041       donor_to_periodic_distance = loc_periodic_global[1] - loc_periodic_global[0];
1042 
1043       PetscCall(PetscSFCreate(PetscObjectComm((PetscObject)dm), &centroidsf));
1044       PetscCall(PetscSFSetGraphFromCoordinates(centroidsf, num_donor, num_periodic, centroid_comps, 1e-10, donor_centroids, periodic_centroids));
1045       PetscCall(PetscSFViewFromOptions(centroidsf, NULL, "-dm_plex_box_label_centroid_sf_view"));
1046       PetscCall(PetscFree2(donor_centroids, periodic_centroids));
1047     }
1048 
1049     { // Create Isoperiodic SF using centroidsSF
1050       PetscInt           pStart, pEnd;
1051       PetscInt          *leaf_faces;
1052       const PetscSFNode *firemote;
1053       PetscSFNode       *isoperiodic_leaves;
1054 
1055       PetscCall(PetscMalloc1(num_periodic, &leaf_faces));
1056       PetscCall(PetscSFBcastBegin(centroidsf, MPIU_INT, donor_faces, leaf_faces, MPI_REPLACE));
1057       PetscCall(PetscSFBcastEnd(centroidsf, MPIU_INT, donor_faces, leaf_faces, MPI_REPLACE));
1058 
1059       PetscCall(PetscMalloc1(num_periodic, &isoperiodic_leaves));
1060       PetscCall(PetscSFGetGraph(centroidsf, NULL, NULL, NULL, &firemote));
1061       for (PetscInt l = 0; l < num_periodic; ++l) {
1062         isoperiodic_leaves[l].index = leaf_faces[l];
1063         isoperiodic_leaves[l].rank  = firemote[l].rank;
1064       }
1065 
1066       PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
1067       PetscCall(PetscSFCreate(PetscObjectComm((PetscObject)dm), &periodicsfs[periodic_sf_index]));
1068       PetscCall(PetscSFSetGraph(periodicsfs[periodic_sf_index], pEnd - pStart, num_periodic, (PetscInt *)periodic_faces, PETSC_COPY_VALUES, isoperiodic_leaves, PETSC_OWN_POINTER));
1069       PetscCall(PetscSFViewFromOptions(periodicsfs[periodic_sf_index], NULL, "-dm_plex_box_label_periodic_sf_view"));
1070       PetscCall(PetscFree(leaf_faces));
1071     }
1072 
1073     transform[periodic_sf_index][0][0] = 1;
1074     transform[periodic_sf_index][1][1] = 1;
1075     transform[periodic_sf_index][2][2] = 1;
1076     transform[periodic_sf_index][3][3] = 1;
1077     transform[periodic_sf_index][d][3] = donor_to_periodic_distance;
1078 
1079     periodic_sf_index++;
1080     PetscCall(PetscSFDestroy(&centroidsf));
1081     if (donor_is) {
1082       PetscCall(ISRestoreIndices(donor_is, &donor_faces));
1083       PetscCall(ISDestroy(&donor_is));
1084     }
1085     if (periodic_is) {
1086       PetscCall(ISRestoreIndices(periodic_is, &periodic_faces));
1087       PetscCall(ISDestroy(&periodic_is));
1088     }
1089     PetscCall(DMClearLabelStratum(dm, "Face Sets", face_pairings[dim - 2][d][0]));
1090     PetscCall(DMClearLabelStratum(dm, "Face Sets", face_pairings[dim - 2][d][1]));
1091   }
1092   PetscCall(DMPlexSetIsoperiodicFaceSF(dm, periodic_sf_index, periodicsfs));
1093   PetscCall(DMPlexSetIsoperiodicFaceTransform(dm, periodic_sf_index, (const PetscScalar *)transform));
1094   for (PetscInt p = 0; p < periodic_sf_index; p++) PetscCall(PetscSFDestroy(&periodicsfs[p]));
1095 
1096   { // Update coordinate DM with new Face Sets label
1097     DM      cdm;
1098     DMLabel oldFaceSets, newFaceSets;
1099     PetscCall(DMGetCoordinateDM(dm, &cdm));
1100     PetscCall(DMGetLabel(cdm, "Face Sets", &oldFaceSets));
1101     if (oldFaceSets) PetscCall(DMRemoveLabelBySelf(cdm, &oldFaceSets, PETSC_FALSE));
1102     PetscCall(DMLabelDuplicate(label, &newFaceSets));
1103     PetscCall(DMAddLabel(cdm, newFaceSets));
1104     PetscCall(DMLabelDestroy(&newFaceSets));
1105   }
1106   PetscFunctionReturn(PETSC_SUCCESS);
1107 }
1108 
1109 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1110 {
1111   DM      boundary, vol;
1112   DMLabel bdlabel;
1113 
1114   PetscFunctionBegin;
1115   PetscAssertPointer(dm, 1);
1116   for (PetscInt i = 0; i < dim; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
1117   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &boundary));
1118   PetscCall(DMSetType(boundary, DMPLEX));
1119   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(boundary, dim, faces, lower, upper, PETSC_FALSE));
1120   PetscCall(DMPlexGenerate(boundary, NULL, interpolate, &vol));
1121   PetscCall(DMGetLabel(vol, "marker", &bdlabel));
1122   if (bdlabel) PetscCall(DMPlexLabelComplete(vol, bdlabel));
1123   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_FALSE, vol));
1124   PetscCall(DMPlexReplace_Internal(dm, &vol));
1125   if (interpolate) {
1126     PetscCall(DMPlexInterpolateInPlace_Internal(dm));
1127     PetscCall(DMPlexSetBoxLabel_Internal(dm, periodicity));
1128   }
1129   PetscCall(DMDestroy(&boundary));
1130   PetscFunctionReturn(PETSC_SUCCESS);
1131 }
1132 
1133 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
1134 {
1135   DMLabel     cutLabel  = NULL;
1136   PetscInt    markerTop = 1, faceMarkerTop = 1;
1137   PetscInt    markerBottom = 1, faceMarkerBottom = 1;
1138   PetscInt    markerFront = 1, faceMarkerFront = 1;
1139   PetscInt    markerBack = 1, faceMarkerBack = 1;
1140   PetscInt    markerRight = 1, faceMarkerRight = 1;
1141   PetscInt    markerLeft = 1, faceMarkerLeft = 1;
1142   PetscInt    dim;
1143   PetscBool   markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
1144   PetscMPIInt rank;
1145 
1146   PetscFunctionBegin;
1147   PetscCall(DMGetDimension(dm, &dim));
1148   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1149   PetscCall(DMCreateLabel(dm, "marker"));
1150   PetscCall(DMCreateLabel(dm, "Face Sets"));
1151   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
1152   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST || bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST || bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
1153     if (cutMarker) {
1154       PetscCall(DMCreateLabel(dm, "periodic_cut"));
1155       PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
1156     }
1157   }
1158   switch (dim) {
1159   case 2:
1160     faceMarkerTop    = 3;
1161     faceMarkerBottom = 1;
1162     faceMarkerRight  = 2;
1163     faceMarkerLeft   = 4;
1164     break;
1165   case 3:
1166     faceMarkerBottom = 1;
1167     faceMarkerTop    = 2;
1168     faceMarkerFront  = 3;
1169     faceMarkerBack   = 4;
1170     faceMarkerRight  = 5;
1171     faceMarkerLeft   = 6;
1172     break;
1173   default:
1174     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
1175   }
1176   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
1177   if (markerSeparate) {
1178     markerBottom = faceMarkerBottom;
1179     markerTop    = faceMarkerTop;
1180     markerFront  = faceMarkerFront;
1181     markerBack   = faceMarkerBack;
1182     markerRight  = faceMarkerRight;
1183     markerLeft   = faceMarkerLeft;
1184   }
1185   {
1186     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
1187     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
1188     const PetscInt numZEdges    = rank == 0 ? edges[2] : 0;
1189     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0] + 1) : 0;
1190     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1] + 1) : 0;
1191     const PetscInt numZVertices = rank == 0 ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2] + 1) : 0;
1192     const PetscInt numCells     = numXEdges * numYEdges * numZEdges;
1193     const PetscInt numXFaces    = numYEdges * numZEdges;
1194     const PetscInt numYFaces    = numXEdges * numZEdges;
1195     const PetscInt numZFaces    = numXEdges * numYEdges;
1196     const PetscInt numTotXFaces = numXVertices * numXFaces;
1197     const PetscInt numTotYFaces = numYVertices * numYFaces;
1198     const PetscInt numTotZFaces = numZVertices * numZFaces;
1199     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
1200     const PetscInt numTotXEdges = numXEdges * numYVertices * numZVertices;
1201     const PetscInt numTotYEdges = numYEdges * numXVertices * numZVertices;
1202     const PetscInt numTotZEdges = numZEdges * numXVertices * numYVertices;
1203     const PetscInt numVertices  = numXVertices * numYVertices * numZVertices;
1204     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
1205     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
1206     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
1207     const PetscInt firstYFace   = firstXFace + numTotXFaces;
1208     const PetscInt firstZFace   = firstYFace + numTotYFaces;
1209     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
1210     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
1211     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
1212     Vec            coordinates;
1213     PetscSection   coordSection;
1214     PetscScalar   *coords;
1215     PetscInt       coordSize;
1216     PetscInt       v, vx, vy, vz;
1217     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
1218 
1219     PetscCall(DMPlexSetChart(dm, 0, numCells + numFaces + numEdges + numVertices));
1220     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
1221     for (f = firstXFace; f < firstXFace + numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
1222     for (e = firstXEdge; e < firstXEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
1223     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
1224     /* Build cells */
1225     for (fz = 0; fz < numZEdges; ++fz) {
1226       for (fy = 0; fy < numYEdges; ++fy) {
1227         for (fx = 0; fx < numXEdges; ++fx) {
1228           PetscInt cell  = (fz * numYEdges + fy) * numXEdges + fx;
1229           PetscInt faceB = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
1230           PetscInt faceT = firstZFace + (fy * numXEdges + fx) * numZVertices + ((fz + 1) % numZVertices);
1231           PetscInt faceF = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
1232           PetscInt faceK = firstYFace + (fz * numXEdges + fx) * numYVertices + ((fy + 1) % numYVertices);
1233           PetscInt faceL = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
1234           PetscInt faceR = firstXFace + (fz * numYEdges + fy) * numXVertices + ((fx + 1) % numXVertices);
1235           /* B,  T,  F,  K,  R,  L */
1236           PetscInt ornt[6] = {-2, 0, 0, -3, 0, -2}; /* ??? */
1237           PetscInt cone[6];
1238 
1239           /* no boundary twisting in 3D */
1240           cone[0] = faceB;
1241           cone[1] = faceT;
1242           cone[2] = faceF;
1243           cone[3] = faceK;
1244           cone[4] = faceR;
1245           cone[5] = faceL;
1246           PetscCall(DMPlexSetCone(dm, cell, cone));
1247           PetscCall(DMPlexSetConeOrientation(dm, cell, ornt));
1248           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
1249           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
1250           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
1251         }
1252       }
1253     }
1254     /* Build x faces */
1255     for (fz = 0; fz < numZEdges; ++fz) {
1256       for (fy = 0; fy < numYEdges; ++fy) {
1257         for (fx = 0; fx < numXVertices; ++fx) {
1258           PetscInt face    = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
1259           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
1260           PetscInt edgeR   = firstZEdge + (((fy + 1) % numYVertices) * numXVertices + fx) * numZEdges + fz;
1261           PetscInt edgeB   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
1262           PetscInt edgeT   = firstYEdge + (((fz + 1) % numZVertices) * numXVertices + fx) * numYEdges + fy;
1263           PetscInt ornt[4] = {0, 0, -1, -1};
1264           PetscInt cone[4];
1265 
1266           if (dim == 3) {
1267             /* markers */
1268             if (bdX != DM_BOUNDARY_PERIODIC) {
1269               if (fx == numXVertices - 1) {
1270                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight));
1271                 PetscCall(DMSetLabelValue(dm, "marker", face, markerRight));
1272               } else if (fx == 0) {
1273                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft));
1274                 PetscCall(DMSetLabelValue(dm, "marker", face, markerLeft));
1275               }
1276             }
1277           }
1278           cone[0] = edgeB;
1279           cone[1] = edgeR;
1280           cone[2] = edgeT;
1281           cone[3] = edgeL;
1282           PetscCall(DMPlexSetCone(dm, face, cone));
1283           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1284         }
1285       }
1286     }
1287     /* Build y faces */
1288     for (fz = 0; fz < numZEdges; ++fz) {
1289       for (fx = 0; fx < numXEdges; ++fx) {
1290         for (fy = 0; fy < numYVertices; ++fy) {
1291           PetscInt face    = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
1292           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
1293           PetscInt edgeR   = firstZEdge + (fy * numXVertices + ((fx + 1) % numXVertices)) * numZEdges + fz;
1294           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1295           PetscInt edgeT   = firstXEdge + (((fz + 1) % numZVertices) * numYVertices + fy) * numXEdges + fx;
1296           PetscInt ornt[4] = {0, 0, -1, -1};
1297           PetscInt cone[4];
1298 
1299           if (dim == 3) {
1300             /* markers */
1301             if (bdY != DM_BOUNDARY_PERIODIC) {
1302               if (fy == numYVertices - 1) {
1303                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack));
1304                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBack));
1305               } else if (fy == 0) {
1306                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront));
1307                 PetscCall(DMSetLabelValue(dm, "marker", face, markerFront));
1308               }
1309             }
1310           }
1311           cone[0] = edgeB;
1312           cone[1] = edgeR;
1313           cone[2] = edgeT;
1314           cone[3] = edgeL;
1315           PetscCall(DMPlexSetCone(dm, face, cone));
1316           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1317         }
1318       }
1319     }
1320     /* Build z faces */
1321     for (fy = 0; fy < numYEdges; ++fy) {
1322       for (fx = 0; fx < numXEdges; ++fx) {
1323         for (fz = 0; fz < numZVertices; fz++) {
1324           PetscInt face    = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
1325           PetscInt edgeL   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
1326           PetscInt edgeR   = firstYEdge + (fz * numXVertices + ((fx + 1) % numXVertices)) * numYEdges + fy;
1327           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1328           PetscInt edgeT   = firstXEdge + (fz * numYVertices + ((fy + 1) % numYVertices)) * numXEdges + fx;
1329           PetscInt ornt[4] = {0, 0, -1, -1};
1330           PetscInt cone[4];
1331 
1332           if (dim == 2) {
1333             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges - 1) {
1334               edgeR += numYEdges - 1 - 2 * fy;
1335               ornt[1] = -1;
1336             }
1337             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges - 1) {
1338               edgeT += numXEdges - 1 - 2 * fx;
1339               ornt[2] = 0;
1340             }
1341             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1342             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1343           } else {
1344             /* markers */
1345             if (bdZ != DM_BOUNDARY_PERIODIC) {
1346               if (fz == numZVertices - 1) {
1347                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop));
1348                 PetscCall(DMSetLabelValue(dm, "marker", face, markerTop));
1349               } else if (fz == 0) {
1350                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom));
1351                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBottom));
1352               }
1353             }
1354           }
1355           cone[0] = edgeB;
1356           cone[1] = edgeR;
1357           cone[2] = edgeT;
1358           cone[3] = edgeL;
1359           PetscCall(DMPlexSetCone(dm, face, cone));
1360           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1361         }
1362       }
1363     }
1364     /* Build Z edges*/
1365     for (vy = 0; vy < numYVertices; vy++) {
1366       for (vx = 0; vx < numXVertices; vx++) {
1367         for (ez = 0; ez < numZEdges; ez++) {
1368           const PetscInt edge    = firstZEdge + (vy * numXVertices + vx) * numZEdges + ez;
1369           const PetscInt vertexB = firstVertex + (ez * numYVertices + vy) * numXVertices + vx;
1370           const PetscInt vertexT = firstVertex + (((ez + 1) % numZVertices) * numYVertices + vy) * numXVertices + vx;
1371           PetscInt       cone[2];
1372 
1373           cone[0] = vertexB;
1374           cone[1] = vertexT;
1375           PetscCall(DMPlexSetCone(dm, edge, cone));
1376           if (dim == 3) {
1377             if (bdX != DM_BOUNDARY_PERIODIC) {
1378               if (vx == numXVertices - 1) {
1379                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1380                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1381                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1382               } else if (vx == 0) {
1383                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1384                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1385                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1386               }
1387             }
1388             if (bdY != DM_BOUNDARY_PERIODIC) {
1389               if (vy == numYVertices - 1) {
1390                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1391                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1392                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1393               } else if (vy == 0) {
1394                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1395                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1396                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1397               }
1398             }
1399           }
1400         }
1401       }
1402     }
1403     /* Build Y edges*/
1404     for (vz = 0; vz < numZVertices; vz++) {
1405       for (vx = 0; vx < numXVertices; vx++) {
1406         for (ey = 0; ey < numYEdges; ey++) {
1407           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges - 1) ? (numXVertices - vx - 1) : (vz * numYVertices + ((ey + 1) % numYVertices)) * numXVertices + vx;
1408           const PetscInt edge    = firstYEdge + (vz * numXVertices + vx) * numYEdges + ey;
1409           const PetscInt vertexF = firstVertex + (vz * numYVertices + ey) * numXVertices + vx;
1410           const PetscInt vertexK = firstVertex + nextv;
1411           PetscInt       cone[2];
1412 
1413           cone[0] = vertexF;
1414           cone[1] = vertexK;
1415           PetscCall(DMPlexSetCone(dm, edge, cone));
1416           if (dim == 2) {
1417             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1418               if (vx == numXVertices - 1) {
1419                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight));
1420                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1421                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1422                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1423               } else if (vx == 0) {
1424                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft));
1425                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1426                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1427                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1428               }
1429             } else {
1430               if (vx == 0 && cutLabel) {
1431                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1432                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1433                 if (ey == numYEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1434               }
1435             }
1436           } else {
1437             if (bdX != DM_BOUNDARY_PERIODIC) {
1438               if (vx == numXVertices - 1) {
1439                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1440                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1441                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1442               } else if (vx == 0) {
1443                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1444                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1445                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1446               }
1447             }
1448             if (bdZ != DM_BOUNDARY_PERIODIC) {
1449               if (vz == numZVertices - 1) {
1450                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1451                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1452                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1453               } else if (vz == 0) {
1454                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1455                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1456                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1457               }
1458             }
1459           }
1460         }
1461       }
1462     }
1463     /* Build X edges*/
1464     for (vz = 0; vz < numZVertices; vz++) {
1465       for (vy = 0; vy < numYVertices; vy++) {
1466         for (ex = 0; ex < numXEdges; ex++) {
1467           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges - 1) ? (numYVertices - vy - 1) * numXVertices : (vz * numYVertices + vy) * numXVertices + (ex + 1) % numXVertices;
1468           const PetscInt edge    = firstXEdge + (vz * numYVertices + vy) * numXEdges + ex;
1469           const PetscInt vertexL = firstVertex + (vz * numYVertices + vy) * numXVertices + ex;
1470           const PetscInt vertexR = firstVertex + nextv;
1471           PetscInt       cone[2];
1472 
1473           cone[0] = vertexL;
1474           cone[1] = vertexR;
1475           PetscCall(DMPlexSetCone(dm, edge, cone));
1476           if (dim == 2) {
1477             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1478               if (vy == numYVertices - 1) {
1479                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop));
1480                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1481                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1482                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1483               } else if (vy == 0) {
1484                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom));
1485                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1486                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1487                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1488               }
1489             } else {
1490               if (vy == 0 && cutLabel) {
1491                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1492                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1493                 if (ex == numXEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1494               }
1495             }
1496           } else {
1497             if (bdY != DM_BOUNDARY_PERIODIC) {
1498               if (vy == numYVertices - 1) {
1499                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1500                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1501                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1502               } else if (vy == 0) {
1503                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1504                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1505                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1506               }
1507             }
1508             if (bdZ != DM_BOUNDARY_PERIODIC) {
1509               if (vz == numZVertices - 1) {
1510                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1511                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1512                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1513               } else if (vz == 0) {
1514                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1515                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1516                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1517               }
1518             }
1519           }
1520         }
1521       }
1522     }
1523     PetscCall(DMPlexSymmetrize(dm));
1524     PetscCall(DMPlexStratify(dm));
1525     /* Build coordinates */
1526     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1527     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1528     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1529     PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVertices));
1530     for (v = firstVertex; v < firstVertex + numVertices; ++v) {
1531       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1532       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1533     }
1534     PetscCall(PetscSectionSetUp(coordSection));
1535     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1536     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1537     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1538     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1539     PetscCall(VecSetBlockSize(coordinates, dim));
1540     PetscCall(VecSetType(coordinates, VECSTANDARD));
1541     PetscCall(VecGetArray(coordinates, &coords));
1542     for (vz = 0; vz < numZVertices; ++vz) {
1543       for (vy = 0; vy < numYVertices; ++vy) {
1544         for (vx = 0; vx < numXVertices; ++vx) {
1545           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * vx;
1546           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * vy;
1547           if (dim == 3) coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 2] = lower[2] + ((upper[2] - lower[2]) / numZEdges) * vz;
1548         }
1549       }
1550     }
1551     PetscCall(VecRestoreArray(coordinates, &coords));
1552     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1553     PetscCall(VecDestroy(&coordinates));
1554   }
1555   PetscFunctionReturn(PETSC_SUCCESS);
1556 }
1557 
1558 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1559 {
1560   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1561   PetscInt       fac[3] = {0, 0, 0}, d;
1562 
1563   PetscFunctionBegin;
1564   PetscAssertPointer(dm, 1);
1565   PetscValidLogicalCollectiveInt(dm, dim, 2);
1566   PetscCall(DMSetDimension(dm, dim));
1567   for (d = 0; d < dim; ++d) {
1568     fac[d] = faces[d];
1569     bdt[d] = periodicity[d];
1570   }
1571   PetscCall(DMPlexCreateCubeMesh_Internal(dm, lower, upper, fac, bdt[0], bdt[1], bdt[2]));
1572   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST || periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST || (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
1573     PetscReal L[3]       = {-1., -1., 0.};
1574     PetscReal maxCell[3] = {-1., -1., 0.};
1575 
1576     for (d = 0; d < dim; ++d) {
1577       if (periodicity[d] != DM_BOUNDARY_NONE) {
1578         L[d]       = upper[d] - lower[d];
1579         maxCell[d] = 1.1 * (L[d] / PetscMax(1, faces[d]));
1580       }
1581     }
1582     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1583   }
1584   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
1585   PetscFunctionReturn(PETSC_SUCCESS);
1586 }
1587 
1588 static PetscErrorCode DMPlexCreateBoxMesh_Internal(DM dm, DMPlexShape shape, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1589 {
1590   PetscFunctionBegin;
1591   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
1592   if (shape == DM_SHAPE_ZBOX) PetscCall(DMPlexCreateBoxMesh_Tensor_SFC_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1593   else if (dim == 1) PetscCall(DMPlexCreateLineMesh_Internal(dm, faces[0], lower[0], upper[0], periodicity[0]));
1594   else if (simplex) PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1595   else PetscCall(DMPlexCreateBoxMesh_Tensor_Internal(dm, dim, faces, lower, upper, periodicity));
1596   if (!interpolate && dim > 1 && !simplex) {
1597     DM udm;
1598 
1599     PetscCall(DMPlexUninterpolate(dm, &udm));
1600     PetscCall(DMPlexCopyCoordinates(dm, udm));
1601     PetscCall(DMPlexReplace_Internal(dm, &udm));
1602   }
1603   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
1604   PetscFunctionReturn(PETSC_SUCCESS);
1605 }
1606 
1607 /*@
1608   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
1609 
1610   Collective
1611 
1612   Input Parameters:
1613 + comm               - The communicator for the `DM` object
1614 . dim                - The spatial dimension
1615 . simplex            - `PETSC_TRUE` for simplices, `PETSC_FALSE` for tensor cells
1616 . faces              - Number of faces per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1617 . lower              - The lower left corner, or `NULL` for (0, 0, 0)
1618 . upper              - The upper right corner, or `NULL` for (1, 1, 1)
1619 . periodicity        - The boundary type for the X,Y,Z direction, or `NULL` for `DM_BOUNDARY_NONE`
1620 . interpolate        - Flag to create intermediate mesh pieces (edges, faces)
1621 . localizationHeight - Flag to localize edges and faces in addition to cells; only significant for periodic meshes
1622 - sparseLocalize     - Flag to localize coordinates only for cells near the periodic boundary; only significant for periodic meshes
1623 
1624   Output Parameter:
1625 . dm - The `DM` object
1626 
1627   Level: beginner
1628 
1629   Note:
1630   To customize this mesh using options, use
1631 .vb
1632   DMCreate(comm, &dm);
1633   DMSetType(dm, DMPLEX);
1634   DMSetFromOptions(dm);
1635 .ve
1636   and use the options in `DMSetFromOptions()`.
1637 
1638   Here is the numbering returned for 2 faces in each direction for tensor cells\:
1639 .vb
1640  10---17---11---18----12
1641   |         |         |
1642   |         |         |
1643  20    2   22    3    24
1644   |         |         |
1645   |         |         |
1646   7---15----8---16----9
1647   |         |         |
1648   |         |         |
1649  19    0   21    1   23
1650   |         |         |
1651   |         |         |
1652   4---13----5---14----6
1653 .ve
1654   and for simplicial cells
1655 .vb
1656  14----8---15----9----16
1657   |\     5  |\      7 |
1658   | \       | \       |
1659  13   2    14    3    15
1660   | 4   \   | 6   \   |
1661   |       \ |       \ |
1662  11----6---12----7----13
1663   |\        |\        |
1664   | \    1  | \     3 |
1665  10   0    11    1    12
1666   | 0   \   | 2   \   |
1667   |       \ |       \ |
1668   8----4----9----5----10
1669 .ve
1670 
1671 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
1672 @*/
1673 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, PetscInt localizationHeight, PetscBool sparseLocalize, DM *dm)
1674 {
1675   PetscInt       fac[3] = {1, 1, 1};
1676   PetscReal      low[3] = {0, 0, 0};
1677   PetscReal      upp[3] = {1, 1, 1};
1678   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1679 
1680   PetscFunctionBegin;
1681   PetscCall(DMCreate(comm, dm));
1682   PetscCall(DMSetType(*dm, DMPLEX));
1683   PetscCall(DMPlexCreateBoxMesh_Internal(*dm, DM_SHAPE_BOX, dim, simplex, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt, interpolate));
1684   if (periodicity) {
1685     DM cdm;
1686 
1687     PetscCall(DMGetCoordinateDM(*dm, &cdm));
1688     PetscCall(DMPlexSetMaxProjectionHeight(cdm, localizationHeight));
1689     PetscCall(DMSetSparseLocalize(*dm, sparseLocalize));
1690     PetscCall(DMLocalizeCoordinates(*dm));
1691   }
1692   PetscFunctionReturn(PETSC_SUCCESS);
1693 }
1694 
1695 static PetscErrorCode DMPlexCreateWedgeBoxMesh_Internal(DM dm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1696 {
1697   DM       bdm, vol;
1698   PetscInt i;
1699 
1700   PetscFunctionBegin;
1701   // TODO Now we can support periodicity
1702   for (i = 0; i < 3; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity not yet supported");
1703   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &bdm));
1704   PetscCall(DMSetType(bdm, DMPLEX));
1705   PetscCall(DMSetDimension(bdm, 2));
1706   PetscCall(PetscLogEventBegin(DMPLEX_Generate, bdm, 0, 0, 0));
1707   PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(bdm, 2, faces, lower, upper, periodicity, PETSC_TRUE));
1708   PetscCall(DMPlexExtrude(bdm, faces[2], upper[2] - lower[2], PETSC_TRUE, PETSC_FALSE, PETSC_FALSE, NULL, NULL, &vol));
1709   PetscCall(PetscLogEventEnd(DMPLEX_Generate, bdm, 0, 0, 0));
1710   PetscCall(DMDestroy(&bdm));
1711   PetscCall(DMPlexReplace_Internal(dm, &vol));
1712   if (lower[2] != 0.0) {
1713     Vec          v;
1714     PetscScalar *x;
1715     PetscInt     cDim, n;
1716 
1717     PetscCall(DMGetCoordinatesLocal(dm, &v));
1718     PetscCall(VecGetBlockSize(v, &cDim));
1719     PetscCall(VecGetLocalSize(v, &n));
1720     PetscCall(VecGetArray(v, &x));
1721     x += cDim;
1722     for (i = 0; i < n; i += cDim) x[i] += lower[2];
1723     PetscCall(VecRestoreArray(v, &x));
1724     PetscCall(DMSetCoordinatesLocal(dm, v));
1725   }
1726   PetscFunctionReturn(PETSC_SUCCESS);
1727 }
1728 
1729 /*@
1730   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tessellating the (x,y) plane and extruding in the third direction using wedge cells.
1731 
1732   Collective
1733 
1734   Input Parameters:
1735 + comm        - The communicator for the `DM` object
1736 . faces       - Number of faces per dimension, or `NULL` for (1, 1, 1)
1737 . lower       - The lower left corner, or `NULL` for (0, 0, 0)
1738 . upper       - The upper right corner, or `NULL` for (1, 1, 1)
1739 . periodicity - The boundary type for the X,Y,Z direction, or `NULL` for `DM_BOUNDARY_NONE`
1740 . orderHeight - If `PETSC_TRUE`, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1741 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1742 
1743   Output Parameter:
1744 . dm - The `DM` object
1745 
1746   Level: beginner
1747 
1748 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateWedgeCylinderMesh()`, `DMExtrude()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1749 @*/
1750 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool orderHeight, PetscBool interpolate, DM *dm)
1751 {
1752   PetscInt       fac[3] = {1, 1, 1};
1753   PetscReal      low[3] = {0, 0, 0};
1754   PetscReal      upp[3] = {1, 1, 1};
1755   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1756 
1757   PetscFunctionBegin;
1758   PetscCall(DMCreate(comm, dm));
1759   PetscCall(DMSetType(*dm, DMPLEX));
1760   PetscCall(DMPlexCreateWedgeBoxMesh_Internal(*dm, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt));
1761   if (!interpolate) {
1762     DM udm;
1763 
1764     PetscCall(DMPlexUninterpolate(*dm, &udm));
1765     PetscCall(DMPlexReplace_Internal(*dm, &udm));
1766   }
1767   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1768   PetscFunctionReturn(PETSC_SUCCESS);
1769 }
1770 
1771 /*
1772   DMPlexTensorPointLexicographic_Private - Returns all tuples of size 'len' with nonnegative integers that are all less than or equal to 'max' for that dimension.
1773 
1774   Input Parameters:
1775 + len - The length of the tuple
1776 . max - The maximum for each dimension, so values are in [0, max)
1777 - tup - A tuple of length len+1: tup[len] > 0 indicates a stopping condition
1778 
1779   Output Parameter:
1780 . tup - A tuple of `len` integers whose entries are at most `max`
1781 
1782   Level: developer
1783 
1784   Note:
1785   Ordering is lexicographic with lowest index as least significant in ordering.
1786   e.g. for len == 2 and max == 2, this will return, in order, {0,0}, {1,0}, {2,0}, {0,1}, {1,1}, {2,1}, {0,2}, {1,2}, {2,2}.
1787 
1788 .seealso: PetscDualSpaceTensorPointLexicographic_Internal(), PetscDualSpaceLatticePointLexicographic_Internal()
1789 */
1790 static PetscErrorCode DMPlexTensorPointLexicographic_Private(PetscInt len, const PetscInt max[], PetscInt tup[])
1791 {
1792   PetscInt i;
1793 
1794   PetscFunctionBegin;
1795   for (i = 0; i < len; ++i) {
1796     if (tup[i] < max[i] - 1) {
1797       break;
1798     } else {
1799       tup[i] = 0;
1800     }
1801   }
1802   if (i == len) tup[i - 1] = max[i - 1];
1803   else ++tup[i];
1804   PetscFunctionReturn(PETSC_SUCCESS);
1805 }
1806 
1807 static PetscInt TupleToIndex_Private(PetscInt len, const PetscInt max[], const PetscInt tup[])
1808 {
1809   PetscInt idx = tup[len - 1];
1810 
1811   for (PetscInt i = len - 2; i >= 0; --i) {
1812     idx *= max[i];
1813     idx += tup[i];
1814   }
1815   return idx;
1816 }
1817 
1818 static void IndexToTuple_Private(PetscInt len, const PetscInt max[], PetscInt idx, PetscInt tup[])
1819 {
1820   for (PetscInt i = 0; i < len; ++i) {
1821     tup[i] = idx % max[i];
1822     idx    = (idx - tup[i]) / max[i];
1823   }
1824 }
1825 
1826 static void TupleToRanks_Private(PetscInt len, const PetscInt max[], const PetscInt procs[], const PetscInt tup[], PetscInt ranks[])
1827 {
1828   for (PetscInt i = 0; i < len; ++i) {
1829     const PetscInt div = max[i] / procs[i];
1830     const PetscInt rem = max[i] % procs[i];
1831     const PetscInt idx = (tup[i] < 0 ? max[i] + tup[i] : tup[i]) % max[i];
1832 
1833     if (idx < rem * (div + 1)) ranks[i] = idx / (div + 1);
1834     else ranks[i] = rem + (idx - rem * (div + 1)) / div;
1835   }
1836 }
1837 
1838 static void RanksToSizes_Private(PetscInt len, const PetscInt max[], const PetscInt procs[], const PetscInt ranks[], PetscInt sizes[])
1839 {
1840   for (PetscInt i = 0; i < len; ++i) {
1841     const PetscInt div = max[i] / procs[i];
1842     const PetscInt rem = max[i] % procs[i];
1843 
1844     sizes[i] = ranks[i] < rem ? div + 1 : div;
1845   }
1846 }
1847 
1848 /*
1849   In serial, the mesh is completely periodic. In parallel, we will include a layer of ghost vertices around the patch, so our edges will not wrap around in parallel. This will instead be handled by the SF.
1850 
1851   The cone for all edges will always be complete. Even in the presence of boundaries, we will keep all support sizes constant. When an edge would not exist in the support, we will create one to wrap back periodically. This allows for uniform stencils, and that edge will not be used for computation.
1852 
1853   All point which do not attain the vertex lower or upper bound in any dimension are owned, the rest are leaves owned by another process and present in the SF.
1854 
1855   Parallel Layout:
1856 
1857   We create a cubic process grid of dimension P^{1/d} on each side. The IndexToTuple_Private() function maps the global rank to the local rank in each dimension. We divide edges using the PETSc distribution rule in each dimension, and then add overlap. TupleToRanks_Private() returns the local rank in ech dimension for a tuple. RanksToSizes_Private() gives the size in each dimension for the domain with those local ranks.
1858 */
1859 static PetscErrorCode DMPlexCreateHypercubicMesh_Internal(DM dm, PetscInt dim, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], PetscInt overlap, const DMBoundaryType bd[])
1860 {
1861   const PetscInt debug = ((DM_Plex *)dm->data)->printAdj;
1862   PetscSF        sf;
1863   Vec            coordinates;
1864   PetscSection   coordSection;
1865   DMLabel        cutLabel    = NULL;
1866   PetscBool      cutMarker   = PETSC_FALSE;
1867   PetscBool      periodic    = PETSC_FALSE;
1868   PetscInt       numCells    = 1;
1869   PetscInt       numVertices = 1;
1870   PetscSFNode   *remotes;
1871   PetscScalar   *coords;
1872   PetscInt      *procs;     // The number of processes along each dimension
1873   PetscInt      *lrank;     // Rank in each dimension, lrank[d] \in [0, procs[d])
1874   PetscInt      *ledges;    // The number of edges along each dimension for this process
1875   PetscInt      *vstart;    // The first vertex along each dimension on this processes
1876   PetscInt      *vertices;  // The number of vertices along each dimension on this process
1877   PetscInt      *rvert;     // The global (not local) vertex number along each dimension
1878   PetscInt      *rrank;     // The rank along each dimension for the process owning rvert[]
1879   PetscInt      *rvertices; // The number of vertices along each dimension for the process rrank[]
1880   PetscInt      *vert, *vtmp, *supp, cone[2], *leaves;
1881   PetscInt       cell = 0, coordSize, Nl = 0, Nl2 = 0;
1882   PetscMPIInt    rank, size;
1883   MPI_Comm       comm;
1884 
1885   PetscFunctionBegin;
1886   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
1887   PetscCallMPI(MPI_Comm_rank(comm, &rank));
1888   PetscCallMPI(MPI_Comm_size(comm, &size));
1889   PetscCall(DMSetDimension(dm, dim));
1890   PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
1891   PetscCall(PetscCalloc4(dim, &procs, dim, &lrank, dim, &rrank, 2 * dim, &supp));
1892   PetscCall(PetscCalloc7(dim, &ledges, dim, &vertices, dim, &rvertices, dim, &vert, dim, &rvert, dim, &vstart, dim, &vtmp));
1893   PetscCall(DMCreateLabel(dm, "marker"));
1894   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
1895   for (PetscInt d = 0; d < dim; ++d) periodic = (periodic || bd[d] == DM_BOUNDARY_PERIODIC) ? PETSC_TRUE : PETSC_FALSE;
1896   if (periodic && cutMarker) {
1897     PetscCall(DMCreateLabel(dm, "periodic_cut"));
1898     PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
1899   }
1900   for (PetscInt d = 0; d < dim; ++d) PetscCheck(bd[d] == DM_BOUNDARY_PERIODIC, comm, PETSC_ERR_SUP, "Hypercubic mesh must be periodic now");
1901   overlap = overlap == PETSC_DETERMINE ? 1 : overlap;
1902   PetscCheck(overlap >= 1, comm, PETSC_ERR_SUP, "Overlap %" PetscInt_FMT " must be greater than 0", overlap);
1903   if (size > 1) {
1904     PetscInt Npr = 1;
1905 
1906     // Make process grid
1907     if (debug) PetscCall(PetscPrintf(comm, "Process grid:"));
1908     for (PetscInt d = 0; d < dim; ++d) {
1909       procs[d] = PetscRintReal(PetscPowReal(size, 1. / dim));
1910       Npr *= procs[d];
1911       if (debug) PetscCall(PetscPrintf(comm, " %" PetscInt_FMT, procs[d]));
1912     }
1913     if (debug) PetscCall(PetscPrintf(comm, "\n"));
1914     PetscCheck(Npr == size, comm, PETSC_ERR_PLIB, "Process grid size %" PetscInt_FMT " != %d comm size", Npr, size);
1915     IndexToTuple_Private(dim, procs, rank, lrank);
1916     for (PetscInt d = 0; d < dim; ++d) {
1917       ledges[d] = edges[d] / procs[d] + (edges[d] % procs[d] > lrank[d] ? 1 : 0);
1918       vstart[d] = 0;
1919       for (PetscInt r = 0; r < lrank[d]; ++r) vstart[d] += edges[d] / procs[d] + (edges[d] % procs[d] > r ? 1 : 0);
1920       vstart[d] -= overlap; // For halo
1921     }
1922   } else {
1923     for (PetscInt d = 0; d < dim; ++d) {
1924       procs[d]  = 1;
1925       ledges[d] = edges[d];
1926     }
1927   }
1928   // Calculate local patch size
1929   for (PetscInt d = 0; d < dim; ++d) {
1930     vertices[d] = ledges[d] + (procs[d] > 1 ? 2 * overlap : 0);
1931     numVertices *= vertices[d];
1932   }
1933   numCells = numVertices * dim;
1934   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1935   for (PetscInt c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, 2));
1936   for (PetscInt v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetSupportSize(dm, v, 2 * dim));
1937   PetscCall(DMSetUp(dm)); /* Allocate space for cones and supports */
1938   /* Build cell cones and vertex supports */
1939   PetscCall(DMCreateLabel(dm, "celltype"));
1940   if (debug) PetscCall(PetscSynchronizedPrintf(comm, "Topology for rank %d:\n", rank));
1941   while (vert[dim - 1] < vertices[dim - 1]) {
1942     const PetscInt vertex = TupleToIndex_Private(dim, vertices, vert) + numCells;
1943     PetscInt       s      = 0;
1944     PetscBool      leaf   = PETSC_FALSE;
1945 
1946     if (debug) {
1947       PetscCall(PetscSynchronizedPrintf(comm, "Vertex %" PetscInt_FMT ":", vertex));
1948       for (PetscInt d = 0; d < dim; ++d) PetscCall(PetscSynchronizedPrintf(comm, " %" PetscInt_FMT, vert[d]));
1949       PetscCall(PetscSynchronizedPrintf(comm, "\n"));
1950     }
1951     PetscCall(DMPlexSetCellType(dm, vertex, DM_POLYTOPE_POINT));
1952     // Define edge cones
1953     for (PetscInt d = 0; d < dim; ++d) {
1954       for (PetscInt e = 0; e < dim; ++e) vtmp[e] = vert[e];
1955       vtmp[d] = (vert[d] + 1) % vertices[d];
1956       cone[0] = vertex;
1957       cone[1] = TupleToIndex_Private(dim, vertices, vtmp) + numCells;
1958       if (debug) {
1959         PetscCall(PetscSynchronizedPrintf(comm, "  Vertex %" PetscInt_FMT ":", cone[1]));
1960         for (PetscInt e = 0; e < dim; ++e) PetscCall(PetscSynchronizedPrintf(comm, " %" PetscInt_FMT, vtmp[e]));
1961         PetscCall(PetscSynchronizedPrintf(comm, "\n"));
1962       }
1963       PetscCall(DMPlexSetCone(dm, cell, cone));
1964       PetscCall(DMPlexSetCellType(dm, cell, DM_POLYTOPE_SEGMENT));
1965       if (debug) PetscCall(PetscSynchronizedPrintf(comm, "  Edge %" PetscInt_FMT " (%" PetscInt_FMT " %" PetscInt_FMT ")\n", cell, cone[0], cone[1]));
1966       ++cell;
1967       // Shared vertices are any in the first or last overlap layers
1968       if (vert[d] < overlap || vert[d] >= vertices[d] - overlap) leaf = PETSC_TRUE;
1969     }
1970     if (size > 1 && leaf) ++Nl;
1971     // Define vertex supports
1972     for (PetscInt d = 0; d < dim; ++d) {
1973       for (PetscInt e = 0; e < dim; ++e) vtmp[e] = vert[e];
1974       vtmp[d]   = (vert[d] + vertices[d] - 1) % vertices[d];
1975       supp[s++] = TupleToIndex_Private(dim, vertices, vtmp) * dim + d;
1976       supp[s++] = (vertex - numCells) * dim + d;
1977       PetscCall(DMPlexSetSupport(dm, vertex, supp));
1978     }
1979     PetscCall(DMPlexTensorPointLexicographic_Private(dim, vertices, vert));
1980   }
1981   if (debug) PetscCall(PetscSynchronizedFlush(comm, NULL));
1982   PetscCall(DMPlexStratify(dm));
1983   // Allocate for SF
1984   PetscCall(PetscMalloc1(Nl, &leaves));
1985   PetscCall(PetscMalloc1(Nl, &remotes));
1986   // Build coordinates
1987   PetscCall(DMGetCoordinateSection(dm, &coordSection));
1988   PetscCall(PetscSectionSetNumFields(coordSection, 1));
1989   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1990   PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1991   for (PetscInt v = numCells; v < numCells + numVertices; ++v) {
1992     PetscCall(PetscSectionSetDof(coordSection, v, dim));
1993     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1994   }
1995   PetscCall(PetscSectionSetUp(coordSection));
1996   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1997   PetscCall(VecCreate(comm, &coordinates));
1998   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1999   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2000   PetscCall(VecSetBlockSize(coordinates, dim));
2001   PetscCall(VecSetType(coordinates, VECSTANDARD));
2002   PetscCall(VecGetArray(coordinates, &coords));
2003   if (debug) PetscCall(PetscSynchronizedPrintf(comm, "Geometry for rank %d:\n", rank));
2004   for (PetscInt d = 0; d < dim; ++d) vert[d] = 0;
2005   while (vert[dim - 1] < vertices[dim - 1]) {
2006     const PetscInt vertex = TupleToIndex_Private(dim, vertices, vert);
2007     PetscBool      leaf   = PETSC_FALSE;
2008 
2009     for (PetscInt d = 0; d < dim; ++d) {
2010       coords[vertex * dim + d] = lower[d] + ((upper[d] - lower[d]) / edges[d]) * (vert[d] + vstart[d]);
2011       if (vert[d] < overlap || vert[d] >= vertices[d] - overlap) leaf = PETSC_TRUE;
2012     }
2013     if (size > 1 && leaf) {
2014       PetscInt rnumCells = 1;
2015 
2016       for (PetscInt d = 0; d < dim; ++d) rvert[d] = vert[d] + vstart[d];
2017       TupleToRanks_Private(dim, edges, procs, rvert, rrank);
2018       leaves[Nl2]       = vertex + numCells;
2019       remotes[Nl2].rank = TupleToIndex_Private(dim, procs, rrank);
2020       RanksToSizes_Private(dim, edges, procs, rrank, rvertices);
2021       for (PetscInt d = 0; d < dim; ++d) {
2022         rvertices[d] += 2 * overlap; // Add halo
2023         rnumCells *= rvertices[d];
2024       }
2025       rnumCells *= dim;
2026       for (PetscInt d = 0; d < dim; ++d) {
2027         const PetscInt diff = rrank[d] - lrank[d];
2028 
2029         if (!diff) rvert[d] = vert[d];                                     // Vertex is local
2030         else if (rvert[d] < 0) rvert[d] = rvertices[d] - 1 + rvert[d];     // Wrap around at the bottom
2031         else if (rvert[d] >= edges[d]) rvert[d] = rvert[d] - edges[d] + 1; // Wrap around at the top
2032         else if (diff == -1) rvert[d] = rvertices[d] - 1 + (vert[d] - overlap);
2033         else if (diff == 1) rvert[d] = (vertices[d] - vert[d] - 1) + overlap;
2034         else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Process distance %" PetscInt_FMT " in direction %" PetscInt_FMT " should not be possible", diff, d);
2035       }
2036       remotes[Nl2].index = TupleToIndex_Private(dim, rvertices, rvert) + rnumCells;
2037       if (debug) PetscCall(PetscSynchronizedPrintf(comm, "Shared Vertex %" PetscInt_FMT " (%" PetscInt_FMT ", %" PetscInt_FMT ")\n", leaves[Nl2], remotes[Nl2].rank, remotes[Nl2].index));
2038       ++Nl2;
2039     }
2040     if (debug) {
2041       PetscCall(PetscSynchronizedPrintf(comm, "Vertex %" PetscInt_FMT ":", vertex));
2042       for (PetscInt d = 0; d < dim; ++d) PetscCall(PetscSynchronizedPrintf(comm, " %" PetscInt_FMT, vert[d] + vstart[d]));
2043       for (PetscInt d = 0; d < dim; ++d) PetscCall(PetscSynchronizedPrintf(comm, " %g", (double)PetscRealPart(coords[vertex * dim + d])));
2044       PetscCall(PetscSynchronizedPrintf(comm, "\n"));
2045     }
2046     PetscCall(DMPlexTensorPointLexicographic_Private(dim, vertices, vert));
2047   }
2048   if (debug) PetscCall(PetscSynchronizedFlush(comm, NULL));
2049   PetscCall(VecRestoreArray(coordinates, &coords));
2050   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2051   PetscCall(VecDestroy(&coordinates));
2052   // Build SF
2053   PetscCheck(Nl == Nl2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Initial number of leaves %" PetscInt_FMT " != %" PetscInt_FMT " final number", Nl, Nl2);
2054   PetscCall(DMGetPointSF(dm, &sf));
2055   PetscCall(PetscSFSetGraph(sf, numCells + numVertices, Nl, leaves, PETSC_OWN_POINTER, remotes, PETSC_OWN_POINTER));
2056   if (debug) PetscCall(PetscSFView(sf, PETSC_VIEWER_STDOUT_WORLD));
2057   //PetscCall(DMSetPeriodicity(dm, NULL, lower, upper));
2058   // Attach the extent
2059   {
2060     PetscContainer c;
2061     PetscInt      *extent, *lextent;
2062 
2063     PetscCall(PetscMalloc1(dim, &extent));
2064     PetscCall(PetscMalloc1(dim, &lextent));
2065     for (PetscInt d = 0; d < dim; ++d) {
2066       extent[d]  = edges[d];
2067       lextent[d] = ledges[d];
2068     }
2069     PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &c));
2070     PetscCall(PetscContainerSetCtxDestroy(c, PetscCtxDestroyDefault));
2071     PetscCall(PetscContainerSetPointer(c, extent));
2072     PetscCall(PetscObjectCompose((PetscObject)dm, "_extent", (PetscObject)c));
2073     PetscCall(PetscContainerDestroy(&c));
2074     PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &c));
2075     PetscCall(PetscContainerSetCtxDestroy(c, PetscCtxDestroyDefault));
2076     PetscCall(PetscContainerSetPointer(c, lextent));
2077     PetscCall(PetscObjectCompose((PetscObject)dm, "_lextent", (PetscObject)c));
2078     PetscCall(PetscContainerDestroy(&c));
2079   }
2080   PetscCall(PetscFree4(procs, lrank, rrank, supp));
2081   PetscCall(PetscFree7(ledges, vertices, rvertices, vert, rvert, vstart, vtmp));
2082   PetscFunctionReturn(PETSC_SUCCESS);
2083 }
2084 
2085 /*@C
2086   DMPlexCreateHypercubicMesh - Creates a periodic mesh on the tensor product of unit intervals using only vertices and edges.
2087 
2088   Collective
2089 
2090   Input Parameters:
2091 + comm    - The communicator for the `DM` object
2092 . dim     - The spatial dimension
2093 . edges   - Number of edges per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
2094 . lower   - The lower left corner, or `NULL` for (0, 0, 0)
2095 . upper   - The upper right corner, or `NULL` for (1, 1, 1)
2096 - overlap - The number of vertices in each direction to include in the overlap (default is 1)
2097 
2098   Output Parameter:
2099 . dm - The DM object
2100 
2101   Level: beginner
2102 
2103   Note:
2104   If you want to customize this mesh using options, you just need to
2105 .vb
2106   DMCreate(comm, &dm);
2107   DMSetType(dm, DMPLEX);
2108   DMSetFromOptions(dm);
2109 .ve
2110   and use the options on the `DMSetFromOptions()` page.
2111 
2112   The vertices are numbered is lexicographic order, and the dim edges exiting a vertex in the positive orthant are number consecutively,
2113 .vb
2114  18--0-19--2-20--4-18
2115   |     |     |     |
2116  13    15    17    13
2117   |     |     |     |
2118  24-12-25-14-26-16-24
2119   |     |     |     |
2120   7     9    11     7
2121   |     |     |     |
2122  21--6-22--8-23-10-21
2123   |     |     |     |
2124   1     3     5     1
2125   |     |     |     |
2126  18--0-19--2-20--4-18
2127 .ve
2128 
2129 .seealso: `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
2130 @*/
2131 PetscErrorCode DMPlexCreateHypercubicMesh(MPI_Comm comm, PetscInt dim, const PetscInt edges[], const PetscReal lower[], const PetscReal upper[], PetscInt overlap, DM *dm)
2132 {
2133   PetscInt       *edg;
2134   PetscReal      *low, *upp;
2135   DMBoundaryType *bdt;
2136   PetscInt        d;
2137 
2138   PetscFunctionBegin;
2139   PetscCall(DMCreate(comm, dm));
2140   PetscCall(DMSetType(*dm, DMPLEX));
2141   PetscCall(PetscMalloc4(dim, &edg, dim, &low, dim, &upp, dim, &bdt));
2142   for (d = 0; d < dim; ++d) {
2143     edg[d] = edges ? edges[d] : 1;
2144     low[d] = lower ? lower[d] : 0.;
2145     upp[d] = upper ? upper[d] : 1.;
2146     bdt[d] = DM_BOUNDARY_PERIODIC;
2147   }
2148   PetscCall(DMPlexCreateHypercubicMesh_Internal(*dm, dim, low, upp, edg, overlap, bdt));
2149   PetscCall(PetscFree4(edg, low, upp, bdt));
2150   PetscFunctionReturn(PETSC_SUCCESS);
2151 }
2152 
2153 /*@
2154   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all `DM` options in the database.
2155 
2156   Logically Collective
2157 
2158   Input Parameters:
2159 + dm     - the `DM` context
2160 - prefix - the prefix to prepend to all option names
2161 
2162   Level: advanced
2163 
2164   Note:
2165   A hyphen (-) must NOT be given at the beginning of the prefix name.
2166   The first character of all runtime options is AUTOMATICALLY the hyphen.
2167 
2168 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `SNESSetFromOptions()`
2169 @*/
2170 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
2171 {
2172   DM_Plex *mesh = (DM_Plex *)dm->data;
2173 
2174   PetscFunctionBegin;
2175   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2176   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, prefix));
2177   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)mesh->partitioner, prefix));
2178   PetscFunctionReturn(PETSC_SUCCESS);
2179 }
2180 
2181 /* Remap geometry to cylinder
2182    TODO: This only works for a single refinement, then it is broken
2183 
2184      Interior square: Linear interpolation is correct
2185      The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
2186      such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
2187 
2188        phi     = arctan(y/x)
2189        d_close = sqrt(1/8 + 1/4 sin^2(phi))
2190        d_far   = sqrt(1/2 + sin^2(phi))
2191 
2192      so we remap them using
2193 
2194        x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
2195        y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
2196 
2197      If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
2198 */
2199 static void snapToCylinder(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
2200 {
2201   const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
2202   const PetscReal ds2 = 0.5 * dis;
2203 
2204   if ((PetscAbsScalar(u[0]) <= ds2) && (PetscAbsScalar(u[1]) <= ds2)) {
2205     f0[0] = u[0];
2206     f0[1] = u[1];
2207   } else {
2208     PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
2209 
2210     x    = PetscRealPart(u[0]);
2211     y    = PetscRealPart(u[1]);
2212     phi  = PetscAtan2Real(y, x);
2213     sinp = PetscSinReal(phi);
2214     cosp = PetscCosReal(phi);
2215     if ((PetscAbsReal(phi) > PETSC_PI / 4.0) && (PetscAbsReal(phi) < 3.0 * PETSC_PI / 4.0)) {
2216       dc = PetscAbsReal(ds2 / sinp);
2217       df = PetscAbsReal(dis / sinp);
2218       xc = ds2 * x / PetscAbsReal(y);
2219       yc = ds2 * PetscSignReal(y);
2220     } else {
2221       dc = PetscAbsReal(ds2 / cosp);
2222       df = PetscAbsReal(dis / cosp);
2223       xc = ds2 * PetscSignReal(x);
2224       yc = ds2 * y / PetscAbsReal(x);
2225     }
2226     f0[0] = xc + (u[0] - xc) * (1.0 - dc) / (df - dc);
2227     f0[1] = yc + (u[1] - yc) * (1.0 - dc) / (df - dc);
2228   }
2229   f0[2] = u[2];
2230 }
2231 
2232 static PetscErrorCode DMPlexCreateHexCylinderMesh_Internal(DM dm, DMBoundaryType periodicZ, PetscInt Nr)
2233 {
2234   const PetscInt dim = 3;
2235   PetscInt       numCells, numVertices;
2236   PetscMPIInt    rank;
2237 
2238   PetscFunctionBegin;
2239   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
2240   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2241   PetscCall(DMSetDimension(dm, dim));
2242   /* Create topology */
2243   {
2244     PetscInt cone[8], c;
2245 
2246     numCells    = rank == 0 ? 5 : 0;
2247     numVertices = rank == 0 ? 16 : 0;
2248     if (periodicZ == DM_BOUNDARY_PERIODIC) {
2249       numCells *= 3;
2250       numVertices = rank == 0 ? 24 : 0;
2251     }
2252     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
2253     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 8));
2254     PetscCall(DMSetUp(dm));
2255     if (rank == 0) {
2256       if (periodicZ == DM_BOUNDARY_PERIODIC) {
2257         cone[0] = 15;
2258         cone[1] = 18;
2259         cone[2] = 17;
2260         cone[3] = 16;
2261         cone[4] = 31;
2262         cone[5] = 32;
2263         cone[6] = 33;
2264         cone[7] = 34;
2265         PetscCall(DMPlexSetCone(dm, 0, cone));
2266         cone[0] = 16;
2267         cone[1] = 17;
2268         cone[2] = 24;
2269         cone[3] = 23;
2270         cone[4] = 32;
2271         cone[5] = 36;
2272         cone[6] = 37;
2273         cone[7] = 33; /* 22 25 26 21 */
2274         PetscCall(DMPlexSetCone(dm, 1, cone));
2275         cone[0] = 18;
2276         cone[1] = 27;
2277         cone[2] = 24;
2278         cone[3] = 17;
2279         cone[4] = 34;
2280         cone[5] = 33;
2281         cone[6] = 37;
2282         cone[7] = 38;
2283         PetscCall(DMPlexSetCone(dm, 2, cone));
2284         cone[0] = 29;
2285         cone[1] = 27;
2286         cone[2] = 18;
2287         cone[3] = 15;
2288         cone[4] = 35;
2289         cone[5] = 31;
2290         cone[6] = 34;
2291         cone[7] = 38;
2292         PetscCall(DMPlexSetCone(dm, 3, cone));
2293         cone[0] = 29;
2294         cone[1] = 15;
2295         cone[2] = 16;
2296         cone[3] = 23;
2297         cone[4] = 35;
2298         cone[5] = 36;
2299         cone[6] = 32;
2300         cone[7] = 31;
2301         PetscCall(DMPlexSetCone(dm, 4, cone));
2302 
2303         cone[0] = 31;
2304         cone[1] = 34;
2305         cone[2] = 33;
2306         cone[3] = 32;
2307         cone[4] = 19;
2308         cone[5] = 22;
2309         cone[6] = 21;
2310         cone[7] = 20;
2311         PetscCall(DMPlexSetCone(dm, 5, cone));
2312         cone[0] = 32;
2313         cone[1] = 33;
2314         cone[2] = 37;
2315         cone[3] = 36;
2316         cone[4] = 22;
2317         cone[5] = 25;
2318         cone[6] = 26;
2319         cone[7] = 21;
2320         PetscCall(DMPlexSetCone(dm, 6, cone));
2321         cone[0] = 34;
2322         cone[1] = 38;
2323         cone[2] = 37;
2324         cone[3] = 33;
2325         cone[4] = 20;
2326         cone[5] = 21;
2327         cone[6] = 26;
2328         cone[7] = 28;
2329         PetscCall(DMPlexSetCone(dm, 7, cone));
2330         cone[0] = 35;
2331         cone[1] = 38;
2332         cone[2] = 34;
2333         cone[3] = 31;
2334         cone[4] = 30;
2335         cone[5] = 19;
2336         cone[6] = 20;
2337         cone[7] = 28;
2338         PetscCall(DMPlexSetCone(dm, 8, cone));
2339         cone[0] = 35;
2340         cone[1] = 31;
2341         cone[2] = 32;
2342         cone[3] = 36;
2343         cone[4] = 30;
2344         cone[5] = 25;
2345         cone[6] = 22;
2346         cone[7] = 19;
2347         PetscCall(DMPlexSetCone(dm, 9, cone));
2348 
2349         cone[0] = 19;
2350         cone[1] = 20;
2351         cone[2] = 21;
2352         cone[3] = 22;
2353         cone[4] = 15;
2354         cone[5] = 16;
2355         cone[6] = 17;
2356         cone[7] = 18;
2357         PetscCall(DMPlexSetCone(dm, 10, cone));
2358         cone[0] = 22;
2359         cone[1] = 21;
2360         cone[2] = 26;
2361         cone[3] = 25;
2362         cone[4] = 16;
2363         cone[5] = 23;
2364         cone[6] = 24;
2365         cone[7] = 17;
2366         PetscCall(DMPlexSetCone(dm, 11, cone));
2367         cone[0] = 20;
2368         cone[1] = 28;
2369         cone[2] = 26;
2370         cone[3] = 21;
2371         cone[4] = 18;
2372         cone[5] = 17;
2373         cone[6] = 24;
2374         cone[7] = 27;
2375         PetscCall(DMPlexSetCone(dm, 12, cone));
2376         cone[0] = 30;
2377         cone[1] = 28;
2378         cone[2] = 20;
2379         cone[3] = 19;
2380         cone[4] = 29;
2381         cone[5] = 15;
2382         cone[6] = 18;
2383         cone[7] = 27;
2384         PetscCall(DMPlexSetCone(dm, 13, cone));
2385         cone[0] = 30;
2386         cone[1] = 19;
2387         cone[2] = 22;
2388         cone[3] = 25;
2389         cone[4] = 29;
2390         cone[5] = 23;
2391         cone[6] = 16;
2392         cone[7] = 15;
2393         PetscCall(DMPlexSetCone(dm, 14, cone));
2394       } else {
2395         cone[0] = 5;
2396         cone[1] = 8;
2397         cone[2] = 7;
2398         cone[3] = 6;
2399         cone[4] = 9;
2400         cone[5] = 12;
2401         cone[6] = 11;
2402         cone[7] = 10;
2403         PetscCall(DMPlexSetCone(dm, 0, cone));
2404         cone[0] = 6;
2405         cone[1] = 7;
2406         cone[2] = 14;
2407         cone[3] = 13;
2408         cone[4] = 12;
2409         cone[5] = 15;
2410         cone[6] = 16;
2411         cone[7] = 11;
2412         PetscCall(DMPlexSetCone(dm, 1, cone));
2413         cone[0] = 8;
2414         cone[1] = 17;
2415         cone[2] = 14;
2416         cone[3] = 7;
2417         cone[4] = 10;
2418         cone[5] = 11;
2419         cone[6] = 16;
2420         cone[7] = 18;
2421         PetscCall(DMPlexSetCone(dm, 2, cone));
2422         cone[0] = 19;
2423         cone[1] = 17;
2424         cone[2] = 8;
2425         cone[3] = 5;
2426         cone[4] = 20;
2427         cone[5] = 9;
2428         cone[6] = 10;
2429         cone[7] = 18;
2430         PetscCall(DMPlexSetCone(dm, 3, cone));
2431         cone[0] = 19;
2432         cone[1] = 5;
2433         cone[2] = 6;
2434         cone[3] = 13;
2435         cone[4] = 20;
2436         cone[5] = 15;
2437         cone[6] = 12;
2438         cone[7] = 9;
2439         PetscCall(DMPlexSetCone(dm, 4, cone));
2440       }
2441     }
2442     PetscCall(DMPlexSymmetrize(dm));
2443     PetscCall(DMPlexStratify(dm));
2444   }
2445   /* Create cube geometry */
2446   {
2447     Vec             coordinates;
2448     PetscSection    coordSection;
2449     PetscScalar    *coords;
2450     PetscInt        coordSize, v;
2451     const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
2452     const PetscReal ds2 = dis / 2.0;
2453 
2454     /* Build coordinates */
2455     PetscCall(DMGetCoordinateSection(dm, &coordSection));
2456     PetscCall(PetscSectionSetNumFields(coordSection, 1));
2457     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
2458     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
2459     for (v = numCells; v < numCells + numVertices; ++v) {
2460       PetscCall(PetscSectionSetDof(coordSection, v, dim));
2461       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
2462     }
2463     PetscCall(PetscSectionSetUp(coordSection));
2464     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2465     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2466     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2467     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2468     PetscCall(VecSetBlockSize(coordinates, dim));
2469     PetscCall(VecSetType(coordinates, VECSTANDARD));
2470     PetscCall(VecGetArray(coordinates, &coords));
2471     if (rank == 0) {
2472       coords[0 * dim + 0]  = -ds2;
2473       coords[0 * dim + 1]  = -ds2;
2474       coords[0 * dim + 2]  = 0.0;
2475       coords[1 * dim + 0]  = ds2;
2476       coords[1 * dim + 1]  = -ds2;
2477       coords[1 * dim + 2]  = 0.0;
2478       coords[2 * dim + 0]  = ds2;
2479       coords[2 * dim + 1]  = ds2;
2480       coords[2 * dim + 2]  = 0.0;
2481       coords[3 * dim + 0]  = -ds2;
2482       coords[3 * dim + 1]  = ds2;
2483       coords[3 * dim + 2]  = 0.0;
2484       coords[4 * dim + 0]  = -ds2;
2485       coords[4 * dim + 1]  = -ds2;
2486       coords[4 * dim + 2]  = 1.0;
2487       coords[5 * dim + 0]  = -ds2;
2488       coords[5 * dim + 1]  = ds2;
2489       coords[5 * dim + 2]  = 1.0;
2490       coords[6 * dim + 0]  = ds2;
2491       coords[6 * dim + 1]  = ds2;
2492       coords[6 * dim + 2]  = 1.0;
2493       coords[7 * dim + 0]  = ds2;
2494       coords[7 * dim + 1]  = -ds2;
2495       coords[7 * dim + 2]  = 1.0;
2496       coords[8 * dim + 0]  = dis;
2497       coords[8 * dim + 1]  = -dis;
2498       coords[8 * dim + 2]  = 0.0;
2499       coords[9 * dim + 0]  = dis;
2500       coords[9 * dim + 1]  = dis;
2501       coords[9 * dim + 2]  = 0.0;
2502       coords[10 * dim + 0] = dis;
2503       coords[10 * dim + 1] = -dis;
2504       coords[10 * dim + 2] = 1.0;
2505       coords[11 * dim + 0] = dis;
2506       coords[11 * dim + 1] = dis;
2507       coords[11 * dim + 2] = 1.0;
2508       coords[12 * dim + 0] = -dis;
2509       coords[12 * dim + 1] = dis;
2510       coords[12 * dim + 2] = 0.0;
2511       coords[13 * dim + 0] = -dis;
2512       coords[13 * dim + 1] = dis;
2513       coords[13 * dim + 2] = 1.0;
2514       coords[14 * dim + 0] = -dis;
2515       coords[14 * dim + 1] = -dis;
2516       coords[14 * dim + 2] = 0.0;
2517       coords[15 * dim + 0] = -dis;
2518       coords[15 * dim + 1] = -dis;
2519       coords[15 * dim + 2] = 1.0;
2520       if (periodicZ == DM_BOUNDARY_PERIODIC) {
2521         /* 15 31 19 */ coords[16 * dim + 0] = -ds2;
2522         coords[16 * dim + 1]                = -ds2;
2523         coords[16 * dim + 2]                = 0.5;
2524         /* 16 32 22 */ coords[17 * dim + 0] = ds2;
2525         coords[17 * dim + 1]                = -ds2;
2526         coords[17 * dim + 2]                = 0.5;
2527         /* 17 33 21 */ coords[18 * dim + 0] = ds2;
2528         coords[18 * dim + 1]                = ds2;
2529         coords[18 * dim + 2]                = 0.5;
2530         /* 18 34 20 */ coords[19 * dim + 0] = -ds2;
2531         coords[19 * dim + 1]                = ds2;
2532         coords[19 * dim + 2]                = 0.5;
2533         /* 29 35 30 */ coords[20 * dim + 0] = -dis;
2534         coords[20 * dim + 1]                = -dis;
2535         coords[20 * dim + 2]                = 0.5;
2536         /* 23 36 25 */ coords[21 * dim + 0] = dis;
2537         coords[21 * dim + 1]                = -dis;
2538         coords[21 * dim + 2]                = 0.5;
2539         /* 24 37 26 */ coords[22 * dim + 0] = dis;
2540         coords[22 * dim + 1]                = dis;
2541         coords[22 * dim + 2]                = 0.5;
2542         /* 27 38 28 */ coords[23 * dim + 0] = -dis;
2543         coords[23 * dim + 1]                = dis;
2544         coords[23 * dim + 2]                = 0.5;
2545       }
2546     }
2547     PetscCall(VecRestoreArray(coordinates, &coords));
2548     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2549     PetscCall(VecDestroy(&coordinates));
2550   }
2551   /* Create periodicity */
2552   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
2553     PetscReal L[3]       = {-1., -1., 0.};
2554     PetscReal maxCell[3] = {-1., -1., 0.};
2555     PetscReal lower[3]   = {0.0, 0.0, 0.0};
2556     PetscReal upper[3]   = {1.0, 1.0, 1.5};
2557     PetscInt  numZCells  = 3;
2558 
2559     L[2]       = upper[2] - lower[2];
2560     maxCell[2] = 1.1 * (L[2] / numZCells);
2561     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
2562   }
2563   {
2564     DM          cdm;
2565     PetscDS     cds;
2566     PetscScalar c[2] = {1.0, 1.0};
2567 
2568     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_TRUE, NULL));
2569     PetscCall(DMGetCoordinateDM(dm, &cdm));
2570     PetscCall(DMGetDS(cdm, &cds));
2571     PetscCall(PetscDSSetConstants(cds, 2, c));
2572   }
2573   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
2574 
2575   /* Wait for coordinate creation before doing in-place modification */
2576   PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2577 
2578   char        oldprefix[PETSC_MAX_PATH_LEN];
2579   const char *prefix;
2580 
2581   PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
2582   PetscCall(PetscStrncpy(oldprefix, prefix, PETSC_MAX_PATH_LEN));
2583   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, "petsc_cyl_ref_"));
2584   for (PetscInt r = 0; r < PetscMax(0, Nr); ++r) {
2585     DM rdm;
2586 
2587     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
2588     PetscCall(DMPlexReplace_Internal(dm, &rdm));
2589   }
2590   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, oldprefix));
2591   PetscCall(DMPlexRemapGeometry(dm, 0.0, snapToCylinder));
2592 
2593   DMLabel         bdlabel, edgelabel;
2594   IS              faceIS;
2595   const PetscInt *faces;
2596   PetscInt        Nf;
2597 
2598   PetscCall(DMCreateLabel(dm, "marker"));
2599   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
2600   PetscCall(DMCreateLabel(dm, "generatrix"));
2601   PetscCall(DMGetLabel(dm, "generatrix", &edgelabel));
2602   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
2603   // Remove faces on top and bottom
2604   PetscCall(DMLabelGetStratumIS(bdlabel, 1, &faceIS));
2605   if (faceIS) {
2606     PetscCall(ISGetLocalSize(faceIS, &Nf));
2607     PetscCall(ISGetIndices(faceIS, &faces));
2608     for (PetscInt f = 0; f < Nf; ++f) {
2609       PetscReal vol, normal[3];
2610 
2611       PetscCall(DMPlexComputeCellGeometryFVM(dm, faces[f], &vol, NULL, normal));
2612       if (PetscAbsReal(normal[2]) < PETSC_SMALL) PetscCall(DMLabelSetValue(edgelabel, faces[f], 1));
2613     }
2614     PetscCall(ISRestoreIndices(faceIS, &faces));
2615     PetscCall(ISDestroy(&faceIS));
2616   }
2617   PetscCall(DMPlexLabelComplete(dm, bdlabel));
2618   PetscCall(DMPlexLabelComplete(dm, edgelabel));
2619   PetscFunctionReturn(PETSC_SUCCESS);
2620 }
2621 
2622 /*@
2623   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
2624 
2625   Collective
2626 
2627   Input Parameters:
2628 + comm      - The communicator for the `DM` object
2629 . periodicZ - The boundary type for the Z direction
2630 - Nr        - The number of refinements to carry out
2631 
2632   Output Parameter:
2633 . dm - The `DM` object
2634 
2635   Level: beginner
2636 
2637   Note:
2638   Here is the output numbering looking from the bottom of the cylinder\:
2639 .vb
2640        17-----14
2641         |     |
2642         |  2  |
2643         |     |
2644  17-----8-----7-----14
2645   |     |     |     |
2646   |  3  |  0  |  1  |
2647   |     |     |     |
2648  19-----5-----6-----13
2649         |     |
2650         |  4  |
2651         |     |
2652        19-----13
2653 
2654  and up through the top
2655 
2656        18-----16
2657         |     |
2658         |  2  |
2659         |     |
2660  18----10----11-----16
2661   |     |     |     |
2662   |  3  |  0  |  1  |
2663   |     |     |     |
2664  20-----9----12-----15
2665         |     |
2666         |  4  |
2667         |     |
2668        20-----15
2669 .ve
2670 
2671 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2672 @*/
2673 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, DMBoundaryType periodicZ, PetscInt Nr, DM *dm)
2674 {
2675   PetscFunctionBegin;
2676   PetscAssertPointer(dm, 4);
2677   PetscCall(DMCreate(comm, dm));
2678   PetscCall(DMSetType(*dm, DMPLEX));
2679   PetscCall(DMPlexCreateHexCylinderMesh_Internal(*dm, periodicZ, Nr));
2680   PetscFunctionReturn(PETSC_SUCCESS);
2681 }
2682 
2683 static PetscErrorCode DMPlexCreateWedgeCylinderMesh_Internal(DM dm, PetscInt n, PetscBool interpolate)
2684 {
2685   const PetscInt dim = 3;
2686   PetscInt       numCells, numVertices, v;
2687   PetscMPIInt    rank;
2688 
2689   PetscFunctionBegin;
2690   PetscCheck(n >= 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %" PetscInt_FMT " cannot be negative", n);
2691   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
2692   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2693   PetscCall(DMSetDimension(dm, dim));
2694   /* Must create the celltype label here so that we do not automatically try to compute the types */
2695   PetscCall(DMCreateLabel(dm, "celltype"));
2696   /* Create topology */
2697   {
2698     PetscInt cone[6], c;
2699 
2700     numCells    = rank == 0 ? n : 0;
2701     numVertices = rank == 0 ? 2 * (n + 1) : 0;
2702     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
2703     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
2704     PetscCall(DMSetUp(dm));
2705     for (c = 0; c < numCells; c++) {
2706       cone[0] = c + n * 1;
2707       cone[1] = (c + 1) % n + n * 1;
2708       cone[2] = 0 + 3 * n;
2709       cone[3] = c + n * 2;
2710       cone[4] = (c + 1) % n + n * 2;
2711       cone[5] = 1 + 3 * n;
2712       PetscCall(DMPlexSetCone(dm, c, cone));
2713       PetscCall(DMPlexSetCellType(dm, c, DM_POLYTOPE_TRI_PRISM_TENSOR));
2714     }
2715     PetscCall(DMPlexSymmetrize(dm));
2716     PetscCall(DMPlexStratify(dm));
2717   }
2718   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetCellType(dm, v, DM_POLYTOPE_POINT));
2719   /* Create cylinder geometry */
2720   {
2721     Vec          coordinates;
2722     PetscSection coordSection;
2723     PetscScalar *coords;
2724     PetscInt     coordSize, c;
2725 
2726     /* Build coordinates */
2727     PetscCall(DMGetCoordinateSection(dm, &coordSection));
2728     PetscCall(PetscSectionSetNumFields(coordSection, 1));
2729     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
2730     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
2731     for (v = numCells; v < numCells + numVertices; ++v) {
2732       PetscCall(PetscSectionSetDof(coordSection, v, dim));
2733       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
2734     }
2735     PetscCall(PetscSectionSetUp(coordSection));
2736     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2737     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2738     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2739     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2740     PetscCall(VecSetBlockSize(coordinates, dim));
2741     PetscCall(VecSetType(coordinates, VECSTANDARD));
2742     PetscCall(VecGetArray(coordinates, &coords));
2743     for (c = 0; c < numCells; c++) {
2744       coords[(c + 0 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
2745       coords[(c + 0 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
2746       coords[(c + 0 * n) * dim + 2] = 1.0;
2747       coords[(c + 1 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
2748       coords[(c + 1 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
2749       coords[(c + 1 * n) * dim + 2] = 0.0;
2750     }
2751     if (rank == 0) {
2752       coords[(2 * n + 0) * dim + 0] = 0.0;
2753       coords[(2 * n + 0) * dim + 1] = 0.0;
2754       coords[(2 * n + 0) * dim + 2] = 1.0;
2755       coords[(2 * n + 1) * dim + 0] = 0.0;
2756       coords[(2 * n + 1) * dim + 1] = 0.0;
2757       coords[(2 * n + 1) * dim + 2] = 0.0;
2758     }
2759     PetscCall(VecRestoreArray(coordinates, &coords));
2760     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2761     PetscCall(VecDestroy(&coordinates));
2762   }
2763   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
2764   /* Interpolate */
2765   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2766   PetscFunctionReturn(PETSC_SUCCESS);
2767 }
2768 
2769 /*@
2770   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
2771 
2772   Collective
2773 
2774   Input Parameters:
2775 + comm        - The communicator for the `DM` object
2776 . n           - The number of wedges around the origin
2777 - interpolate - Create edges and faces
2778 
2779   Output Parameter:
2780 . dm - The `DM` object
2781 
2782   Level: beginner
2783 
2784 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2785 @*/
2786 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
2787 {
2788   PetscFunctionBegin;
2789   PetscAssertPointer(dm, 4);
2790   PetscCall(DMCreate(comm, dm));
2791   PetscCall(DMSetType(*dm, DMPLEX));
2792   PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(*dm, n, interpolate));
2793   PetscFunctionReturn(PETSC_SUCCESS);
2794 }
2795 
2796 static inline PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2797 {
2798   PetscReal prod = 0.0;
2799   PetscInt  i;
2800   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
2801   return PetscSqrtReal(prod);
2802 }
2803 
2804 static inline PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2805 {
2806   PetscReal prod = 0.0;
2807   PetscInt  i;
2808   for (i = 0; i < dim; ++i) prod += x[i] * y[i];
2809   return prod;
2810 }
2811 
2812 /* The first constant is the sphere radius */
2813 static void snapToSphere(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
2814 {
2815   PetscReal r     = PetscRealPart(constants[0]);
2816   PetscReal norm2 = 0.0, fac;
2817   PetscInt  n     = uOff[1] - uOff[0], d;
2818 
2819   for (d = 0; d < n; ++d) norm2 += PetscSqr(PetscRealPart(u[d]));
2820   fac = r / PetscSqrtReal(norm2);
2821   for (d = 0; d < n; ++d) f0[d] = u[d] * fac;
2822 }
2823 
2824 static PetscErrorCode DMPlexCreateSphereMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, PetscReal R)
2825 {
2826   const PetscInt embedDim = dim + 1;
2827   PetscSection   coordSection;
2828   Vec            coordinates;
2829   PetscScalar   *coords;
2830   PetscReal     *coordsIn;
2831   PetscInt       numCells, numEdges, numVerts = 0, firstVertex = 0, v, firstEdge, coordSize, d, e;
2832   PetscMPIInt    rank;
2833 
2834   PetscFunctionBegin;
2835   PetscValidLogicalCollectiveBool(dm, simplex, 3);
2836   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
2837   PetscCall(DMSetDimension(dm, dim));
2838   PetscCall(DMSetCoordinateDim(dm, dim + 1));
2839   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2840   switch (dim) {
2841   case 1:
2842     numCells = 16;
2843     numVerts = numCells;
2844 
2845     // Build Topology
2846     PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2847     for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2848     PetscCall(DMSetUp(dm));
2849     for (PetscInt c = 0; c < numCells; ++c) {
2850       PetscInt cone[2];
2851 
2852       cone[0] = c + numCells;
2853       cone[1] = (c + 1) % numVerts + numCells;
2854       PetscCall(DMPlexSetCone(dm, c, cone));
2855     }
2856     PetscCall(DMPlexSymmetrize(dm));
2857     PetscCall(DMPlexStratify(dm));
2858     PetscCall(PetscMalloc1(numVerts * embedDim, &coordsIn));
2859     for (PetscInt v = 0; v < numVerts; ++v) {
2860       const PetscReal rad = 2. * PETSC_PI * v / numVerts;
2861 
2862       coordsIn[v * embedDim + 0] = PetscCosReal(rad);
2863       coordsIn[v * embedDim + 1] = PetscSinReal(rad);
2864     }
2865     break;
2866   case 2:
2867     if (simplex) {
2868       const PetscReal radius    = PetscSqrtReal(1 + PETSC_PHI * PETSC_PHI) / (1.0 + PETSC_PHI);
2869       const PetscReal edgeLen   = 2.0 / (1.0 + PETSC_PHI) * (R / radius);
2870       const PetscInt  degree    = 5;
2871       PetscReal       vertex[3] = {0.0, 1.0 / (1.0 + PETSC_PHI), PETSC_PHI / (1.0 + PETSC_PHI)};
2872       PetscInt        s[3]      = {1, 1, 1};
2873       PetscInt        cone[3];
2874       PetscInt       *graph;
2875 
2876       vertex[0] *= R / radius;
2877       vertex[1] *= R / radius;
2878       vertex[2] *= R / radius;
2879       numCells    = rank == 0 ? 20 : 0;
2880       numVerts    = rank == 0 ? 12 : 0;
2881       firstVertex = numCells;
2882       /* Use icosahedron, which for a R-sphere has coordinates which are all cyclic permutations of
2883 
2884            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
2885 
2886          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2887          length is then given by 2/(1+\phi) = 2 * 0.38197 = 0.76393.
2888       */
2889       /* Construct vertices */
2890       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2891       if (rank == 0) {
2892         for (PetscInt p = 0, i = 0; p < embedDim; ++p) {
2893           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2894             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2895               for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertex[(d + p) % embedDim];
2896               ++i;
2897             }
2898           }
2899         }
2900       }
2901       /* Construct graph */
2902       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2903       for (PetscInt i = 0; i < numVerts; ++i) {
2904         PetscInt k = 0;
2905         for (PetscInt j = 0; j < numVerts; ++j) {
2906           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2907             graph[i * numVerts + j] = 1;
2908             ++k;
2909           }
2910         }
2911         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid icosahedron, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2912       }
2913       /* Build Topology */
2914       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2915       for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2916       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2917       /* Cells */
2918       for (PetscInt i = 0, c = 0; i < numVerts; ++i) {
2919         for (PetscInt j = 0; j < i; ++j) {
2920           for (PetscInt k = 0; k < j; ++k) {
2921             if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i]) {
2922               cone[0] = firstVertex + i;
2923               cone[1] = firstVertex + j;
2924               cone[2] = firstVertex + k;
2925               /* Check orientation */
2926               {
2927                 const PetscInt epsilon[3][3][3] = {
2928                   {{0, 0, 0},  {0, 0, 1},  {0, -1, 0}},
2929                   {{0, 0, -1}, {0, 0, 0},  {1, 0, 0} },
2930                   {{0, 1, 0},  {-1, 0, 0}, {0, 0, 0} }
2931                 };
2932                 PetscReal normal[3];
2933                 PetscInt  e, f;
2934 
2935                 for (d = 0; d < embedDim; ++d) {
2936                   normal[d] = 0.0;
2937                   for (e = 0; e < embedDim; ++e) {
2938                     for (f = 0; f < embedDim; ++f) normal[d] += epsilon[d][e][f] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]);
2939                   }
2940                 }
2941                 if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2942                   PetscInt tmp = cone[1];
2943                   cone[1]      = cone[2];
2944                   cone[2]      = tmp;
2945                 }
2946               }
2947               PetscCall(DMPlexSetCone(dm, c++, cone));
2948             }
2949           }
2950         }
2951       }
2952       PetscCall(DMPlexSymmetrize(dm));
2953       PetscCall(DMPlexStratify(dm));
2954       PetscCall(PetscFree(graph));
2955     } else {
2956       /*
2957         12-21--13
2958          |     |
2959         25  4  24
2960          |     |
2961   12-25--9-16--8-24--13
2962    |     |     |     |
2963   23  5 17  0 15  3  22
2964    |     |     |     |
2965   10-20--6-14--7-19--11
2966          |     |
2967         20  1  19
2968          |     |
2969         10-18--11
2970          |     |
2971         23  2  22
2972          |     |
2973         12-21--13
2974        */
2975       PetscInt cone[4], ornt[4];
2976 
2977       numCells    = rank == 0 ? 6 : 0;
2978       numEdges    = rank == 0 ? 12 : 0;
2979       numVerts    = rank == 0 ? 8 : 0;
2980       firstVertex = numCells;
2981       firstEdge   = numCells + numVerts;
2982       /* Build Topology */
2983       PetscCall(DMPlexSetChart(dm, 0, numCells + numEdges + numVerts));
2984       for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 4));
2985       for (e = firstEdge; e < firstEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
2986       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2987       if (rank == 0) {
2988         /* Cell 0 */
2989         cone[0] = 14;
2990         cone[1] = 15;
2991         cone[2] = 16;
2992         cone[3] = 17;
2993         PetscCall(DMPlexSetCone(dm, 0, cone));
2994         ornt[0] = 0;
2995         ornt[1] = 0;
2996         ornt[2] = 0;
2997         ornt[3] = 0;
2998         PetscCall(DMPlexSetConeOrientation(dm, 0, ornt));
2999         /* Cell 1 */
3000         cone[0] = 18;
3001         cone[1] = 19;
3002         cone[2] = 14;
3003         cone[3] = 20;
3004         PetscCall(DMPlexSetCone(dm, 1, cone));
3005         ornt[0] = 0;
3006         ornt[1] = 0;
3007         ornt[2] = -1;
3008         ornt[3] = 0;
3009         PetscCall(DMPlexSetConeOrientation(dm, 1, ornt));
3010         /* Cell 2 */
3011         cone[0] = 21;
3012         cone[1] = 22;
3013         cone[2] = 18;
3014         cone[3] = 23;
3015         PetscCall(DMPlexSetCone(dm, 2, cone));
3016         ornt[0] = 0;
3017         ornt[1] = 0;
3018         ornt[2] = -1;
3019         ornt[3] = 0;
3020         PetscCall(DMPlexSetConeOrientation(dm, 2, ornt));
3021         /* Cell 3 */
3022         cone[0] = 19;
3023         cone[1] = 22;
3024         cone[2] = 24;
3025         cone[3] = 15;
3026         PetscCall(DMPlexSetCone(dm, 3, cone));
3027         ornt[0] = -1;
3028         ornt[1] = -1;
3029         ornt[2] = 0;
3030         ornt[3] = -1;
3031         PetscCall(DMPlexSetConeOrientation(dm, 3, ornt));
3032         /* Cell 4 */
3033         cone[0] = 16;
3034         cone[1] = 24;
3035         cone[2] = 21;
3036         cone[3] = 25;
3037         PetscCall(DMPlexSetCone(dm, 4, cone));
3038         ornt[0] = -1;
3039         ornt[1] = -1;
3040         ornt[2] = -1;
3041         ornt[3] = 0;
3042         PetscCall(DMPlexSetConeOrientation(dm, 4, ornt));
3043         /* Cell 5 */
3044         cone[0] = 20;
3045         cone[1] = 17;
3046         cone[2] = 25;
3047         cone[3] = 23;
3048         PetscCall(DMPlexSetCone(dm, 5, cone));
3049         ornt[0] = -1;
3050         ornt[1] = -1;
3051         ornt[2] = -1;
3052         ornt[3] = -1;
3053         PetscCall(DMPlexSetConeOrientation(dm, 5, ornt));
3054         /* Edges */
3055         cone[0] = 6;
3056         cone[1] = 7;
3057         PetscCall(DMPlexSetCone(dm, 14, cone));
3058         cone[0] = 7;
3059         cone[1] = 8;
3060         PetscCall(DMPlexSetCone(dm, 15, cone));
3061         cone[0] = 8;
3062         cone[1] = 9;
3063         PetscCall(DMPlexSetCone(dm, 16, cone));
3064         cone[0] = 9;
3065         cone[1] = 6;
3066         PetscCall(DMPlexSetCone(dm, 17, cone));
3067         cone[0] = 10;
3068         cone[1] = 11;
3069         PetscCall(DMPlexSetCone(dm, 18, cone));
3070         cone[0] = 11;
3071         cone[1] = 7;
3072         PetscCall(DMPlexSetCone(dm, 19, cone));
3073         cone[0] = 6;
3074         cone[1] = 10;
3075         PetscCall(DMPlexSetCone(dm, 20, cone));
3076         cone[0] = 12;
3077         cone[1] = 13;
3078         PetscCall(DMPlexSetCone(dm, 21, cone));
3079         cone[0] = 13;
3080         cone[1] = 11;
3081         PetscCall(DMPlexSetCone(dm, 22, cone));
3082         cone[0] = 10;
3083         cone[1] = 12;
3084         PetscCall(DMPlexSetCone(dm, 23, cone));
3085         cone[0] = 13;
3086         cone[1] = 8;
3087         PetscCall(DMPlexSetCone(dm, 24, cone));
3088         cone[0] = 12;
3089         cone[1] = 9;
3090         PetscCall(DMPlexSetCone(dm, 25, cone));
3091       }
3092       PetscCall(DMPlexSymmetrize(dm));
3093       PetscCall(DMPlexStratify(dm));
3094       /* Build coordinates */
3095       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
3096       if (rank == 0) {
3097         coordsIn[0 * embedDim + 0] = -R;
3098         coordsIn[0 * embedDim + 1] = R;
3099         coordsIn[0 * embedDim + 2] = -R;
3100         coordsIn[1 * embedDim + 0] = R;
3101         coordsIn[1 * embedDim + 1] = R;
3102         coordsIn[1 * embedDim + 2] = -R;
3103         coordsIn[2 * embedDim + 0] = R;
3104         coordsIn[2 * embedDim + 1] = -R;
3105         coordsIn[2 * embedDim + 2] = -R;
3106         coordsIn[3 * embedDim + 0] = -R;
3107         coordsIn[3 * embedDim + 1] = -R;
3108         coordsIn[3 * embedDim + 2] = -R;
3109         coordsIn[4 * embedDim + 0] = -R;
3110         coordsIn[4 * embedDim + 1] = R;
3111         coordsIn[4 * embedDim + 2] = R;
3112         coordsIn[5 * embedDim + 0] = R;
3113         coordsIn[5 * embedDim + 1] = R;
3114         coordsIn[5 * embedDim + 2] = R;
3115         coordsIn[6 * embedDim + 0] = -R;
3116         coordsIn[6 * embedDim + 1] = -R;
3117         coordsIn[6 * embedDim + 2] = R;
3118         coordsIn[7 * embedDim + 0] = R;
3119         coordsIn[7 * embedDim + 1] = -R;
3120         coordsIn[7 * embedDim + 2] = R;
3121       }
3122     }
3123     break;
3124   case 3:
3125     if (simplex) {
3126       const PetscReal edgeLen         = 1.0 / PETSC_PHI;
3127       PetscReal       vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
3128       PetscReal       vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
3129       PetscReal       vertexC[4]      = {0.5, 0.5 * PETSC_PHI, 0.5 / PETSC_PHI, 0.0};
3130       const PetscInt  degree          = 12;
3131       PetscInt        s[4]            = {1, 1, 1};
3132       PetscInt        evenPerm[12][4] = {
3133         {0, 1, 2, 3},
3134         {0, 2, 3, 1},
3135         {0, 3, 1, 2},
3136         {1, 0, 3, 2},
3137         {1, 2, 0, 3},
3138         {1, 3, 2, 0},
3139         {2, 0, 1, 3},
3140         {2, 1, 3, 0},
3141         {2, 3, 0, 1},
3142         {3, 0, 2, 1},
3143         {3, 1, 0, 2},
3144         {3, 2, 1, 0}
3145       };
3146       PetscInt  cone[4];
3147       PetscInt *graph, p, i, j, k, l;
3148 
3149       vertexA[0] *= R;
3150       vertexA[1] *= R;
3151       vertexA[2] *= R;
3152       vertexA[3] *= R;
3153       vertexB[0] *= R;
3154       vertexB[1] *= R;
3155       vertexB[2] *= R;
3156       vertexB[3] *= R;
3157       vertexC[0] *= R;
3158       vertexC[1] *= R;
3159       vertexC[2] *= R;
3160       vertexC[3] *= R;
3161       numCells    = rank == 0 ? 600 : 0;
3162       numVerts    = rank == 0 ? 120 : 0;
3163       firstVertex = numCells;
3164       /* Use the 600-cell, which for a unit sphere has coordinates which are
3165 
3166            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
3167                (\pm 1,    0,       0,      0)  all cyclic permutations   8
3168            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
3169 
3170          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
3171          length is then given by 1/\phi = 0.61803.
3172 
3173          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
3174          http://mathworld.wolfram.com/600-Cell.html
3175       */
3176       /* Construct vertices */
3177       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
3178       i = 0;
3179       if (rank == 0) {
3180         for (s[0] = -1; s[0] < 2; s[0] += 2) {
3181           for (s[1] = -1; s[1] < 2; s[1] += 2) {
3182             for (s[2] = -1; s[2] < 2; s[2] += 2) {
3183               for (s[3] = -1; s[3] < 2; s[3] += 2) {
3184                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[d] * vertexA[d];
3185                 ++i;
3186               }
3187             }
3188           }
3189         }
3190         for (p = 0; p < embedDim; ++p) {
3191           s[1] = s[2] = s[3] = 1;
3192           for (s[0] = -1; s[0] < 2; s[0] += 2) {
3193             for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertexB[(d + p) % embedDim];
3194             ++i;
3195           }
3196         }
3197         for (p = 0; p < 12; ++p) {
3198           s[3] = 1;
3199           for (s[0] = -1; s[0] < 2; s[0] += 2) {
3200             for (s[1] = -1; s[1] < 2; s[1] += 2) {
3201               for (s[2] = -1; s[2] < 2; s[2] += 2) {
3202                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[evenPerm[p][d]] * vertexC[evenPerm[p][d]];
3203                 ++i;
3204               }
3205             }
3206           }
3207         }
3208       }
3209       PetscCheck(i == numVerts, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertices %" PetscInt_FMT " != %" PetscInt_FMT, i, numVerts);
3210       /* Construct graph */
3211       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
3212       for (i = 0; i < numVerts; ++i) {
3213         for (j = 0, k = 0; j < numVerts; ++j) {
3214           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
3215             graph[i * numVerts + j] = 1;
3216             ++k;
3217           }
3218         }
3219         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
3220       }
3221       /* Build Topology */
3222       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
3223       for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
3224       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
3225       /* Cells */
3226       if (rank == 0) {
3227         for (PetscInt i = 0, c = 0; i < numVerts; ++i) {
3228           for (j = 0; j < i; ++j) {
3229             for (k = 0; k < j; ++k) {
3230               for (l = 0; l < k; ++l) {
3231                 if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i] && graph[l * numVerts + i] && graph[l * numVerts + j] && graph[l * numVerts + k]) {
3232                   cone[0] = firstVertex + i;
3233                   cone[1] = firstVertex + j;
3234                   cone[2] = firstVertex + k;
3235                   cone[3] = firstVertex + l;
3236                   /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
3237                   {
3238                     const PetscInt epsilon[4][4][4][4] = {
3239                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, -1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 0, 0}, {0, 1, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 1, 0}, {0, -1, 0, 0}, {0, 0, 0, 0}}},
3240 
3241                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}}, {{0, 0, -1, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}}},
3242 
3243                       {{{0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, 0, 0}, {0, -1, 0, 0}}, {{0, 0, 0, -1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 1, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}},
3244 
3245                       {{{0, 0, 0, 0}, {0, 0, -1, 0}, {0, 1, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 1, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}}, {{0, -1, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}} }
3246                     };
3247                     PetscReal normal[4];
3248                     PetscInt  e, f, g;
3249 
3250                     for (d = 0; d < embedDim; ++d) {
3251                       normal[d] = 0.0;
3252                       for (e = 0; e < embedDim; ++e) {
3253                         for (f = 0; f < embedDim; ++f) {
3254                           for (g = 0; g < embedDim; ++g) {
3255                             normal[d] += epsilon[d][e][f][g] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]) * (coordsIn[l * embedDim + f] - coordsIn[i * embedDim + f]);
3256                           }
3257                         }
3258                       }
3259                     }
3260                     if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
3261                       PetscInt tmp = cone[1];
3262                       cone[1]      = cone[2];
3263                       cone[2]      = tmp;
3264                     }
3265                   }
3266                   PetscCall(DMPlexSetCone(dm, c++, cone));
3267                 }
3268               }
3269             }
3270           }
3271         }
3272       }
3273       PetscCall(DMPlexSymmetrize(dm));
3274       PetscCall(DMPlexStratify(dm));
3275       PetscCall(PetscFree(graph));
3276     }
3277     break;
3278   default:
3279     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unsupported dimension for sphere: %" PetscInt_FMT, dim);
3280   }
3281   /* Create coordinates */
3282   PetscCall(DMGetCoordinateSection(dm, &coordSection));
3283   PetscCall(PetscSectionSetNumFields(coordSection, 1));
3284   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, embedDim));
3285   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVerts));
3286   for (v = firstVertex; v < firstVertex + numVerts; ++v) {
3287     PetscCall(PetscSectionSetDof(coordSection, v, embedDim));
3288     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, embedDim));
3289   }
3290   PetscCall(PetscSectionSetUp(coordSection));
3291   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
3292   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
3293   PetscCall(VecSetBlockSize(coordinates, embedDim));
3294   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
3295   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
3296   PetscCall(VecSetType(coordinates, VECSTANDARD));
3297   PetscCall(VecGetArray(coordinates, &coords));
3298   for (v = 0; v < numVerts; ++v)
3299     for (d = 0; d < embedDim; ++d) coords[v * embedDim + d] = coordsIn[v * embedDim + d];
3300   PetscCall(VecRestoreArray(coordinates, &coords));
3301   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
3302   PetscCall(VecDestroy(&coordinates));
3303   PetscCall(PetscFree(coordsIn));
3304   {
3305     DM          cdm;
3306     PetscDS     cds;
3307     PetscScalar c = R;
3308 
3309     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_TRUE, snapToSphere));
3310     PetscCall(DMGetCoordinateDM(dm, &cdm));
3311     PetscCall(DMGetDS(cdm, &cds));
3312     PetscCall(PetscDSSetConstants(cds, 1, &c));
3313   }
3314   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
3315   /* Wait for coordinate creation before doing in-place modification */
3316   if (simplex) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
3317   PetscFunctionReturn(PETSC_SUCCESS);
3318 }
3319 
3320 typedef void (*TPSEvaluateFunc)(const PetscReal[], PetscReal *, PetscReal[], PetscReal (*)[3]);
3321 
3322 /*
3323  The Schwarz P implicit surface is
3324 
3325      f(x) = cos(x0) + cos(x1) + cos(x2) = 0
3326 */
3327 static void TPSEvaluate_SchwarzP(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
3328 {
3329   PetscReal c[3] = {PetscCosReal(y[0] * PETSC_PI), PetscCosReal(y[1] * PETSC_PI), PetscCosReal(y[2] * PETSC_PI)};
3330   PetscReal g[3] = {-PetscSinReal(y[0] * PETSC_PI), -PetscSinReal(y[1] * PETSC_PI), -PetscSinReal(y[2] * PETSC_PI)};
3331   f[0]           = c[0] + c[1] + c[2];
3332   for (PetscInt i = 0; i < 3; i++) {
3333     grad[i] = PETSC_PI * g[i];
3334     for (PetscInt j = 0; j < 3; j++) hess[i][j] = (i == j) ? -PetscSqr(PETSC_PI) * c[i] : 0.;
3335   }
3336 }
3337 
3338 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
3339 static PetscErrorCode TPSExtrudeNormalFunc_SchwarzP(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
3340 {
3341   for (PetscInt i = 0; i < 3; i++) u[i] = -PETSC_PI * PetscSinReal(x[i] * PETSC_PI);
3342   return PETSC_SUCCESS;
3343 }
3344 
3345 /*
3346  The Gyroid implicit surface is
3347 
3348  f(x,y,z) = sin(pi * x) * cos (pi * (y + 1/2))  + sin(pi * (y + 1/2)) * cos(pi * (z + 1/4)) + sin(pi * (z + 1/4)) * cos(pi * x)
3349 
3350 */
3351 static void TPSEvaluate_Gyroid(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
3352 {
3353   PetscReal s[3] = {PetscSinReal(PETSC_PI * y[0]), PetscSinReal(PETSC_PI * (y[1] + .5)), PetscSinReal(PETSC_PI * (y[2] + .25))};
3354   PetscReal c[3] = {PetscCosReal(PETSC_PI * y[0]), PetscCosReal(PETSC_PI * (y[1] + .5)), PetscCosReal(PETSC_PI * (y[2] + .25))};
3355   f[0]           = s[0] * c[1] + s[1] * c[2] + s[2] * c[0];
3356   grad[0]        = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
3357   grad[1]        = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
3358   grad[2]        = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
3359   hess[0][0]     = -PetscSqr(PETSC_PI) * (s[0] * c[1] + s[2] * c[0]);
3360   hess[0][1]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
3361   hess[0][2]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
3362   hess[1][0]     = -PetscSqr(PETSC_PI) * (s[1] * c[2] + s[0] * c[1]);
3363   hess[1][1]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
3364   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
3365   hess[2][0]     = -PetscSqr(PETSC_PI) * (s[2] * c[0] + s[1] * c[2]);
3366   hess[2][1]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
3367   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
3368 }
3369 
3370 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
3371 static PetscErrorCode TPSExtrudeNormalFunc_Gyroid(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
3372 {
3373   PetscReal s[3] = {PetscSinReal(PETSC_PI * x[0]), PetscSinReal(PETSC_PI * (x[1] + .5)), PetscSinReal(PETSC_PI * (x[2] + .25))};
3374   PetscReal c[3] = {PetscCosReal(PETSC_PI * x[0]), PetscCosReal(PETSC_PI * (x[1] + .5)), PetscCosReal(PETSC_PI * (x[2] + .25))};
3375   u[0]           = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
3376   u[1]           = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
3377   u[2]           = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
3378   return PETSC_SUCCESS;
3379 }
3380 
3381 /*
3382    We wish to solve
3383 
3384          min_y || y - x ||^2  subject to f(y) = 0
3385 
3386    Let g(y) = grad(f).  The minimization problem is equivalent to asking to satisfy
3387    f(y) = 0 and (y-x) is parallel to g(y).  We do this by using Householder QR to obtain a basis for the
3388    tangent space and ask for both components in the tangent space to be zero.
3389 
3390    Take g to be a column vector and compute the "full QR" factorization Q R = g,
3391    where Q = I - 2 n n^T is a symmetric orthogonal matrix.
3392    The first column of Q is parallel to g so the remaining two columns span the null space.
3393    Let Qn = Q[:,1:] be those remaining columns.  Then Qn Qn^T is an orthogonal projector into the tangent space.
3394    Since Q is symmetric, this is equivalent to multiplying by Q and taking the last two entries.
3395    In total, we have a system of 3 equations in 3 unknowns:
3396 
3397      f(y) = 0                       1 equation
3398      Qn^T (y - x) = 0               2 equations
3399 
3400    Here, we compute the residual and Jacobian of this system.
3401 */
3402 static void TPSNearestPointResJac(TPSEvaluateFunc feval, const PetscScalar x[], const PetscScalar y[], PetscScalar res[], PetscScalar J[])
3403 {
3404   PetscReal yreal[3] = {PetscRealPart(y[0]), PetscRealPart(y[1]), PetscRealPart(y[2])};
3405   PetscReal d[3]     = {PetscRealPart(y[0] - x[0]), PetscRealPart(y[1] - x[1]), PetscRealPart(y[2] - x[2])};
3406   PetscReal f, grad[3], n[3], norm, norm_y[3], nd, nd_y[3], sign;
3407   PetscReal n_y[3][3] = {
3408     {0, 0, 0},
3409     {0, 0, 0},
3410     {0, 0, 0}
3411   };
3412 
3413   feval(yreal, &f, grad, n_y);
3414 
3415   for (PetscInt i = 0; i < 3; i++) n[i] = grad[i];
3416   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
3417   for (PetscInt i = 0; i < 3; i++) norm_y[i] = 1. / norm * n[i] * n_y[i][i];
3418 
3419   // Define the Householder reflector
3420   sign = n[0] >= 0 ? 1. : -1.;
3421   n[0] += norm * sign;
3422   for (PetscInt i = 0; i < 3; i++) n_y[0][i] += norm_y[i] * sign;
3423 
3424   norm      = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
3425   norm_y[0] = 1. / norm * (n[0] * n_y[0][0]);
3426   norm_y[1] = 1. / norm * (n[0] * n_y[0][1] + n[1] * n_y[1][1]);
3427   norm_y[2] = 1. / norm * (n[0] * n_y[0][2] + n[2] * n_y[2][2]);
3428 
3429   for (PetscInt i = 0; i < 3; i++) {
3430     n[i] /= norm;
3431     for (PetscInt j = 0; j < 3; j++) {
3432       // note that n[i] is n_old[i]/norm when executing the code below
3433       n_y[i][j] = n_y[i][j] / norm - n[i] / norm * norm_y[j];
3434     }
3435   }
3436 
3437   nd = n[0] * d[0] + n[1] * d[1] + n[2] * d[2];
3438   for (PetscInt i = 0; i < 3; i++) nd_y[i] = n[i] + n_y[0][i] * d[0] + n_y[1][i] * d[1] + n_y[2][i] * d[2];
3439 
3440   res[0] = f;
3441   res[1] = d[1] - 2 * n[1] * nd;
3442   res[2] = d[2] - 2 * n[2] * nd;
3443   // J[j][i] is J_{ij} (column major)
3444   for (PetscInt j = 0; j < 3; j++) {
3445     J[0 + j * 3] = grad[j];
3446     J[1 + j * 3] = (j == 1) * 1. - 2 * (n_y[1][j] * nd + n[1] * nd_y[j]);
3447     J[2 + j * 3] = (j == 2) * 1. - 2 * (n_y[2][j] * nd + n[2] * nd_y[j]);
3448   }
3449 }
3450 
3451 /*
3452    Project x to the nearest point on the implicit surface using Newton's method.
3453 */
3454 static PetscErrorCode TPSNearestPoint(TPSEvaluateFunc feval, PetscScalar x[])
3455 {
3456   PetscScalar y[3] = {x[0], x[1], x[2]}; // Initial guess
3457 
3458   PetscFunctionBegin;
3459   for (PetscInt iter = 0; iter < 10; iter++) {
3460     PetscScalar res[3], J[9];
3461     PetscReal   resnorm;
3462     TPSNearestPointResJac(feval, x, y, res, J);
3463     resnorm = PetscSqrtReal(PetscSqr(PetscRealPart(res[0])) + PetscSqr(PetscRealPart(res[1])) + PetscSqr(PetscRealPart(res[2])));
3464     if (0) { // Turn on this monitor if you need to confirm quadratic convergence
3465       PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%" PetscInt_FMT "] res [%g %g %g]\n", iter, (double)PetscRealPart(res[0]), (double)PetscRealPart(res[1]), (double)PetscRealPart(res[2])));
3466     }
3467     if (resnorm < PETSC_SMALL) break;
3468 
3469     // Take the Newton step
3470     PetscCall(PetscKernel_A_gets_inverse_A_3(J, 0., PETSC_FALSE, NULL));
3471     PetscKernel_v_gets_v_minus_A_times_w_3(y, J, res);
3472   }
3473   for (PetscInt i = 0; i < 3; i++) x[i] = y[i];
3474   PetscFunctionReturn(PETSC_SUCCESS);
3475 }
3476 
3477 const char *const DMPlexTPSTypes[] = {"SCHWARZ_P", "GYROID", "DMPlexTPSType", "DMPLEX_TPS_", NULL};
3478 
3479 static PetscErrorCode DMPlexCreateTPSMesh_Internal(DM dm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness)
3480 {
3481   PetscMPIInt rank;
3482   PetscInt    topoDim = 2, spaceDim = 3, numFaces = 0, numVertices = 0, numEdges = 0;
3483   PetscInt(*edges)[2] = NULL, *edgeSets = NULL;
3484   PetscInt           *cells_flat = NULL;
3485   PetscReal          *vtxCoords  = NULL;
3486   TPSEvaluateFunc     evalFunc   = NULL;
3487   PetscSimplePointFn *normalFunc = NULL;
3488   DMLabel             label;
3489 
3490   PetscFunctionBegin;
3491   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
3492   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3493   PetscCheck((layers != 0) ^ (thickness == 0.), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Layers %" PetscInt_FMT " must be nonzero iff thickness %g is nonzero", layers, (double)thickness);
3494   switch (tpstype) {
3495   case DMPLEX_TPS_SCHWARZ_P:
3496     PetscCheck(!periodic || (periodic[0] == DM_BOUNDARY_NONE && periodic[1] == DM_BOUNDARY_NONE && periodic[2] == DM_BOUNDARY_NONE), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Schwarz P does not support periodic meshes");
3497     if (rank == 0) {
3498       PetscInt(*cells)[6][4][4] = NULL; // [junction, junction-face, cell, conn]
3499       PetscInt  Njunctions = 0, Ncuts = 0, Npipes[3], vcount;
3500       PetscReal L = 1;
3501 
3502       Npipes[0]   = (extent[0] + 1) * extent[1] * extent[2];
3503       Npipes[1]   = extent[0] * (extent[1] + 1) * extent[2];
3504       Npipes[2]   = extent[0] * extent[1] * (extent[2] + 1);
3505       Njunctions  = extent[0] * extent[1] * extent[2];
3506       Ncuts       = 2 * (extent[0] * extent[1] + extent[1] * extent[2] + extent[2] * extent[0]);
3507       numVertices = 4 * (Npipes[0] + Npipes[1] + Npipes[2]) + 8 * Njunctions;
3508       PetscCall(PetscMalloc1(3 * numVertices, &vtxCoords));
3509       PetscCall(PetscMalloc1(Njunctions, &cells));
3510       PetscCall(PetscMalloc1(Ncuts * 4, &edges));
3511       PetscCall(PetscMalloc1(Ncuts * 4, &edgeSets));
3512       // x-normal pipes
3513       vcount = 0;
3514       for (PetscInt i = 0; i < extent[0] + 1; i++) {
3515         for (PetscInt j = 0; j < extent[1]; j++) {
3516           for (PetscInt k = 0; k < extent[2]; k++) {
3517             for (PetscInt l = 0; l < 4; l++) {
3518               vtxCoords[vcount++] = (2 * i - 1) * L;
3519               vtxCoords[vcount++] = 2 * j * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3520               vtxCoords[vcount++] = 2 * k * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3521             }
3522           }
3523         }
3524       }
3525       // y-normal pipes
3526       for (PetscInt i = 0; i < extent[0]; i++) {
3527         for (PetscInt j = 0; j < extent[1] + 1; j++) {
3528           for (PetscInt k = 0; k < extent[2]; k++) {
3529             for (PetscInt l = 0; l < 4; l++) {
3530               vtxCoords[vcount++] = 2 * i * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3531               vtxCoords[vcount++] = (2 * j - 1) * L;
3532               vtxCoords[vcount++] = 2 * k * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3533             }
3534           }
3535         }
3536       }
3537       // z-normal pipes
3538       for (PetscInt i = 0; i < extent[0]; i++) {
3539         for (PetscInt j = 0; j < extent[1]; j++) {
3540           for (PetscInt k = 0; k < extent[2] + 1; k++) {
3541             for (PetscInt l = 0; l < 4; l++) {
3542               vtxCoords[vcount++] = 2 * i * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3543               vtxCoords[vcount++] = 2 * j * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3544               vtxCoords[vcount++] = (2 * k - 1) * L;
3545             }
3546           }
3547         }
3548       }
3549       // junctions
3550       for (PetscInt i = 0; i < extent[0]; i++) {
3551         for (PetscInt j = 0; j < extent[1]; j++) {
3552           for (PetscInt k = 0; k < extent[2]; k++) {
3553             const PetscInt J = (i * extent[1] + j) * extent[2] + k, Jvoff = (Npipes[0] + Npipes[1] + Npipes[2]) * 4 + J * 8;
3554             PetscCheck(vcount / 3 == Jvoff, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected vertex count");
3555             for (PetscInt ii = 0; ii < 2; ii++) {
3556               for (PetscInt jj = 0; jj < 2; jj++) {
3557                 for (PetscInt kk = 0; kk < 2; kk++) {
3558                   double Ls           = (1 - sqrt(2) / 4) * L;
3559                   vtxCoords[vcount++] = 2 * i * L + (2 * ii - 1) * Ls;
3560                   vtxCoords[vcount++] = 2 * j * L + (2 * jj - 1) * Ls;
3561                   vtxCoords[vcount++] = 2 * k * L + (2 * kk - 1) * Ls;
3562                 }
3563               }
3564             }
3565             const PetscInt jfaces[3][2][4] = {
3566               {{3, 1, 0, 2}, {7, 5, 4, 6}}, // x-aligned
3567               {{5, 4, 0, 1}, {7, 6, 2, 3}}, // y-aligned
3568               {{6, 2, 0, 4}, {7, 3, 1, 5}}  // z-aligned
3569             };
3570             const PetscInt pipe_lo[3] = {// vertex numbers of pipes
3571                                          ((i * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + Npipes[0] + Npipes[1]) * 4};
3572             const PetscInt pipe_hi[3] = {// vertex numbers of pipes
3573                                          (((i + 1) * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j + 1) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + 1 + Npipes[0] + Npipes[1]) * 4};
3574             for (PetscInt dir = 0; dir < 3; dir++) { // x,y,z
3575               const PetscInt ijk[3] = {i, j, k};
3576               for (PetscInt l = 0; l < 4; l++) { // rotations
3577                 cells[J][dir * 2 + 0][l][0] = pipe_lo[dir] + l;
3578                 cells[J][dir * 2 + 0][l][1] = Jvoff + jfaces[dir][0][l];
3579                 cells[J][dir * 2 + 0][l][2] = Jvoff + jfaces[dir][0][(l - 1 + 4) % 4];
3580                 cells[J][dir * 2 + 0][l][3] = pipe_lo[dir] + (l - 1 + 4) % 4;
3581                 cells[J][dir * 2 + 1][l][0] = Jvoff + jfaces[dir][1][l];
3582                 cells[J][dir * 2 + 1][l][1] = pipe_hi[dir] + l;
3583                 cells[J][dir * 2 + 1][l][2] = pipe_hi[dir] + (l - 1 + 4) % 4;
3584                 cells[J][dir * 2 + 1][l][3] = Jvoff + jfaces[dir][1][(l - 1 + 4) % 4];
3585                 if (ijk[dir] == 0) {
3586                   edges[numEdges][0] = pipe_lo[dir] + l;
3587                   edges[numEdges][1] = pipe_lo[dir] + (l + 1) % 4;
3588                   edgeSets[numEdges] = dir * 2 + 1;
3589                   numEdges++;
3590                 }
3591                 if (ijk[dir] + 1 == extent[dir]) {
3592                   edges[numEdges][0] = pipe_hi[dir] + l;
3593                   edges[numEdges][1] = pipe_hi[dir] + (l + 1) % 4;
3594                   edgeSets[numEdges] = dir * 2 + 2;
3595                   numEdges++;
3596                 }
3597               }
3598             }
3599           }
3600         }
3601       }
3602       PetscCheck(numEdges == Ncuts * 4, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Edge count %" PetscInt_FMT " incompatible with number of cuts %" PetscInt_FMT, numEdges, Ncuts);
3603       numFaces   = 24 * Njunctions;
3604       cells_flat = cells[0][0][0];
3605     }
3606     evalFunc   = TPSEvaluate_SchwarzP;
3607     normalFunc = TPSExtrudeNormalFunc_SchwarzP;
3608     break;
3609   case DMPLEX_TPS_GYROID:
3610     if (rank == 0) {
3611       // This is a coarse mesh approximation of the gyroid shifted to being the zero of the level set
3612       //
3613       //     sin(pi*x)*cos(pi*(y+1/2)) + sin(pi*(y+1/2))*cos(pi*(z+1/4)) + sin(pi*(z+1/4))*cos(x)
3614       //
3615       // on the cell [0,2]^3.
3616       //
3617       // Think about dividing that cell into four columns, and focus on the column [0,1]x[0,1]x[0,2].
3618       // If you looked at the gyroid in that column at different slices of z you would see that it kind of spins
3619       // like a boomerang:
3620       //
3621       //     z = 0          z = 1/4        z = 1/2        z = 3/4     //
3622       //     -----          -------        -------        -------     //
3623       //                                                              //
3624       //     +       +      +       +      +       +      +   \   +   //
3625       //      \                                   /            \      //
3626       //       \            `-_   _-'            /              }     //
3627       //        *-_            `-'            _-'              /      //
3628       //     +     `-+      +       +      +-'     +      +   /   +   //
3629       //                                                              //
3630       //                                                              //
3631       //     z = 1          z = 5/4        z = 3/2        z = 7/4     //
3632       //     -----          -------        -------        -------     //
3633       //                                                              //
3634       //     +-_     +      +       +      +     _-+      +   /   +   //
3635       //        `-_            _-_            _-`            /        //
3636       //           \        _-'   `-_        /              {         //
3637       //            \                       /                \        //
3638       //     +       +      +       +      +       +      +   \   +   //
3639       //
3640       //
3641       // This course mesh approximates each of these slices by two line segments,
3642       // and then connects the segments in consecutive layers with quadrilateral faces.
3643       // All of the end points of the segments are multiples of 1/4 except for the
3644       // point * in the picture for z = 0 above and the similar points in other layers.
3645       // That point is at (gamma, gamma, 0), where gamma is calculated below.
3646       //
3647       // The column  [1,2]x[1,2]x[0,2] looks the same as this column;
3648       // The columns [1,2]x[0,1]x[0,2] and [0,1]x[1,2]x[0,2] are mirror images.
3649       //
3650       // As for how this method turned into the names given to the vertices:
3651       // that was not systematic, it was just the way it worked out in my handwritten notes.
3652 
3653       PetscInt facesPerBlock = 64;
3654       PetscInt vertsPerBlock = 56;
3655       PetscInt extentPlus[3];
3656       PetscInt numBlocks, numBlocksPlus;
3657       const PetscInt A = 0, B = 1, C = 2, D = 3, E = 4, F = 5, G = 6, H = 7, II = 8, J = 9, K = 10, L = 11, M = 12, N = 13, O = 14, P = 15, Q = 16, R = 17, S = 18, T = 19, U = 20, V = 21, W = 22, X = 23, Y = 24, Z = 25, Ap = 26, Bp = 27, Cp = 28, Dp = 29, Ep = 30, Fp = 31, Gp = 32, Hp = 33, Ip = 34, Jp = 35, Kp = 36, Lp = 37, Mp = 38, Np = 39, Op = 40, Pp = 41, Qp = 42, Rp = 43, Sp = 44, Tp = 45, Up = 46, Vp = 47, Wp = 48, Xp = 49, Yp = 50, Zp = 51, Aq = 52, Bq = 53, Cq = 54, Dq = 55;
3658       const PetscInt pattern[64][4] = {
3659         /* face to vertex within the coarse discretization of a single gyroid block */
3660         /* layer 0 */
3661         {A,           C,           K,           G          },
3662         {C,           B,           II,          K          },
3663         {D,           A,           H,           L          },
3664         {B + 56 * 1,  D,           L,           J          },
3665         {E,           B + 56 * 1,  J,           N          },
3666         {A + 56 * 2,  E,           N,           H + 56 * 2 },
3667         {F,           A + 56 * 2,  G + 56 * 2,  M          },
3668         {B,           F,           M,           II         },
3669         /* layer 1 */
3670         {G,           K,           Q,           O          },
3671         {K,           II,          P,           Q          },
3672         {L,           H,           O + 56 * 1,  R          },
3673         {J,           L,           R,           P          },
3674         {N,           J,           P,           S          },
3675         {H + 56 * 2,  N,           S,           O + 56 * 3 },
3676         {M,           G + 56 * 2,  O + 56 * 2,  T          },
3677         {II,          M,           T,           P          },
3678         /* layer 2 */
3679         {O,           Q,           Y,           U          },
3680         {Q,           P,           W,           Y          },
3681         {R,           O + 56 * 1,  U + 56 * 1,  Ap         },
3682         {P,           R,           Ap,          W          },
3683         {S,           P,           X,           Bp         },
3684         {O + 56 * 3,  S,           Bp,          V + 56 * 1 },
3685         {T,           O + 56 * 2,  V,           Z          },
3686         {P,           T,           Z,           X          },
3687         /* layer 3 */
3688         {U,           Y,           Ep,          Dp         },
3689         {Y,           W,           Cp,          Ep         },
3690         {Ap,          U + 56 * 1,  Dp + 56 * 1, Gp         },
3691         {W,           Ap,          Gp,          Cp         },
3692         {Bp,          X,           Cp + 56 * 2, Fp         },
3693         {V + 56 * 1,  Bp,          Fp,          Dp + 56 * 1},
3694         {Z,           V,           Dp,          Hp         },
3695         {X,           Z,           Hp,          Cp + 56 * 2},
3696         /* layer 4 */
3697         {Dp,          Ep,          Mp,          Kp         },
3698         {Ep,          Cp,          Ip,          Mp         },
3699         {Gp,          Dp + 56 * 1, Lp,          Np         },
3700         {Cp,          Gp,          Np,          Jp         },
3701         {Fp,          Cp + 56 * 2, Jp + 56 * 2, Pp         },
3702         {Dp + 56 * 1, Fp,          Pp,          Lp         },
3703         {Hp,          Dp,          Kp,          Op         },
3704         {Cp + 56 * 2, Hp,          Op,          Ip + 56 * 2},
3705         /* layer 5 */
3706         {Kp,          Mp,          Sp,          Rp         },
3707         {Mp,          Ip,          Qp,          Sp         },
3708         {Np,          Lp,          Rp,          Tp         },
3709         {Jp,          Np,          Tp,          Qp + 56 * 1},
3710         {Pp,          Jp + 56 * 2, Qp + 56 * 3, Up         },
3711         {Lp,          Pp,          Up,          Rp         },
3712         {Op,          Kp,          Rp,          Vp         },
3713         {Ip + 56 * 2, Op,          Vp,          Qp + 56 * 2},
3714         /* layer 6 */
3715         {Rp,          Sp,          Aq,          Yp         },
3716         {Sp,          Qp,          Wp,          Aq         },
3717         {Tp,          Rp,          Yp,          Cq         },
3718         {Qp + 56 * 1, Tp,          Cq,          Wp + 56 * 1},
3719         {Up,          Qp + 56 * 3, Xp + 56 * 1, Dq         },
3720         {Rp,          Up,          Dq,          Zp         },
3721         {Vp,          Rp,          Zp,          Bq         },
3722         {Qp + 56 * 2, Vp,          Bq,          Xp         },
3723         /* layer 7 (the top is the periodic image of the bottom of layer 0) */
3724         {Yp,          Aq,          C + 56 * 4,  A + 56 * 4 },
3725         {Aq,          Wp,          B + 56 * 4,  C + 56 * 4 },
3726         {Cq,          Yp,          A + 56 * 4,  D + 56 * 4 },
3727         {Wp + 56 * 1, Cq,          D + 56 * 4,  B + 56 * 5 },
3728         {Dq,          Xp + 56 * 1, B + 56 * 5,  E + 56 * 4 },
3729         {Zp,          Dq,          E + 56 * 4,  A + 56 * 6 },
3730         {Bq,          Zp,          A + 56 * 6,  F + 56 * 4 },
3731         {Xp,          Bq,          F + 56 * 4,  B + 56 * 4 }
3732       };
3733       const PetscReal gamma                = PetscAcosReal((PetscSqrtReal(3.) - 1.) / PetscSqrtReal(2.)) / PETSC_PI;
3734       const PetscReal patternCoords[56][3] = {
3735         {1.,        0.,        0.  }, /* A  */
3736         {0.,        1.,        0.  }, /* B  */
3737         {gamma,     gamma,     0.  }, /* C  */
3738         {1 + gamma, 1 - gamma, 0.  }, /* D  */
3739         {2 - gamma, 2 - gamma, 0.  }, /* E  */
3740         {1 - gamma, 1 + gamma, 0.  }, /* F  */
3741 
3742         {.5,        0,         .25 }, /* G  */
3743         {1.5,       0.,        .25 }, /* H  */
3744         {.5,        1.,        .25 }, /* II */
3745         {1.5,       1.,        .25 }, /* J  */
3746         {.25,       .5,        .25 }, /* K  */
3747         {1.25,      .5,        .25 }, /* L  */
3748         {.75,       1.5,       .25 }, /* M  */
3749         {1.75,      1.5,       .25 }, /* N  */
3750 
3751         {0.,        0.,        .5  }, /* O  */
3752         {1.,        1.,        .5  }, /* P  */
3753         {gamma,     1 - gamma, .5  }, /* Q  */
3754         {1 + gamma, gamma,     .5  }, /* R  */
3755         {2 - gamma, 1 + gamma, .5  }, /* S  */
3756         {1 - gamma, 2 - gamma, .5  }, /* T  */
3757 
3758         {0.,        .5,        .75 }, /* U  */
3759         {0.,        1.5,       .75 }, /* V  */
3760         {1.,        .5,        .75 }, /* W  */
3761         {1.,        1.5,       .75 }, /* X  */
3762         {.5,        .75,       .75 }, /* Y  */
3763         {.5,        1.75,      .75 }, /* Z  */
3764         {1.5,       .25,       .75 }, /* Ap */
3765         {1.5,       1.25,      .75 }, /* Bp */
3766 
3767         {1.,        0.,        1.  }, /* Cp */
3768         {0.,        1.,        1.  }, /* Dp */
3769         {1 - gamma, 1 - gamma, 1.  }, /* Ep */
3770         {1 + gamma, 1 + gamma, 1.  }, /* Fp */
3771         {2 - gamma, gamma,     1.  }, /* Gp */
3772         {gamma,     2 - gamma, 1.  }, /* Hp */
3773 
3774         {.5,        0.,        1.25}, /* Ip */
3775         {1.5,       0.,        1.25}, /* Jp */
3776         {.5,        1.,        1.25}, /* Kp */
3777         {1.5,       1.,        1.25}, /* Lp */
3778         {.75,       .5,        1.25}, /* Mp */
3779         {1.75,      .5,        1.25}, /* Np */
3780         {.25,       1.5,       1.25}, /* Op */
3781         {1.25,      1.5,       1.25}, /* Pp */
3782 
3783         {0.,        0.,        1.5 }, /* Qp */
3784         {1.,        1.,        1.5 }, /* Rp */
3785         {1 - gamma, gamma,     1.5 }, /* Sp */
3786         {2 - gamma, 1 - gamma, 1.5 }, /* Tp */
3787         {1 + gamma, 2 - gamma, 1.5 }, /* Up */
3788         {gamma,     1 + gamma, 1.5 }, /* Vp */
3789 
3790         {0.,        .5,        1.75}, /* Wp */
3791         {0.,        1.5,       1.75}, /* Xp */
3792         {1.,        .5,        1.75}, /* Yp */
3793         {1.,        1.5,       1.75}, /* Zp */
3794         {.5,        .25,       1.75}, /* Aq */
3795         {.5,        1.25,      1.75}, /* Bq */
3796         {1.5,       .75,       1.75}, /* Cq */
3797         {1.5,       1.75,      1.75}, /* Dq */
3798       };
3799       PetscInt(*cells)[64][4] = NULL;
3800       PetscBool *seen;
3801       PetscInt  *vertToTrueVert;
3802       PetscInt   count;
3803 
3804       for (PetscInt i = 0; i < 3; i++) extentPlus[i] = extent[i] + 1;
3805       numBlocks = 1;
3806       for (PetscInt i = 0; i < 3; i++) numBlocks *= extent[i];
3807       numBlocksPlus = 1;
3808       for (PetscInt i = 0; i < 3; i++) numBlocksPlus *= extentPlus[i];
3809       numFaces = numBlocks * facesPerBlock;
3810       PetscCall(PetscMalloc1(numBlocks, &cells));
3811       PetscCall(PetscCalloc1(numBlocksPlus * vertsPerBlock, &seen));
3812       for (PetscInt k = 0; k < extent[2]; k++) {
3813         for (PetscInt j = 0; j < extent[1]; j++) {
3814           for (PetscInt i = 0; i < extent[0]; i++) {
3815             for (PetscInt f = 0; f < facesPerBlock; f++) {
3816               for (PetscInt v = 0; v < 4; v++) {
3817                 PetscInt vertRaw     = pattern[f][v];
3818                 PetscInt blockidx    = vertRaw / 56;
3819                 PetscInt patternvert = vertRaw % 56;
3820                 PetscInt xplus       = (blockidx & 1);
3821                 PetscInt yplus       = (blockidx & 2) >> 1;
3822                 PetscInt zplus       = (blockidx & 4) >> 2;
3823                 PetscInt zcoord      = (periodic && periodic[2] == DM_BOUNDARY_PERIODIC) ? ((k + zplus) % extent[2]) : (k + zplus);
3824                 PetscInt ycoord      = (periodic && periodic[1] == DM_BOUNDARY_PERIODIC) ? ((j + yplus) % extent[1]) : (j + yplus);
3825                 PetscInt xcoord      = (periodic && periodic[0] == DM_BOUNDARY_PERIODIC) ? ((i + xplus) % extent[0]) : (i + xplus);
3826                 PetscInt vert        = ((zcoord * extentPlus[1] + ycoord) * extentPlus[0] + xcoord) * 56 + patternvert;
3827 
3828                 cells[(k * extent[1] + j) * extent[0] + i][f][v] = vert;
3829                 seen[vert]                                       = PETSC_TRUE;
3830               }
3831             }
3832           }
3833         }
3834       }
3835       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++)
3836         if (seen[i]) numVertices++;
3837       count = 0;
3838       PetscCall(PetscMalloc1(numBlocksPlus * vertsPerBlock, &vertToTrueVert));
3839       PetscCall(PetscMalloc1(numVertices * 3, &vtxCoords));
3840       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) vertToTrueVert[i] = -1;
3841       for (PetscInt k = 0; k < extentPlus[2]; k++) {
3842         for (PetscInt j = 0; j < extentPlus[1]; j++) {
3843           for (PetscInt i = 0; i < extentPlus[0]; i++) {
3844             for (PetscInt v = 0; v < vertsPerBlock; v++) {
3845               PetscInt vIdx = ((k * extentPlus[1] + j) * extentPlus[0] + i) * vertsPerBlock + v;
3846 
3847               if (seen[vIdx]) {
3848                 PetscInt thisVert;
3849 
3850                 vertToTrueVert[vIdx] = thisVert = count++;
3851 
3852                 for (PetscInt d = 0; d < 3; d++) vtxCoords[3 * thisVert + d] = patternCoords[v][d];
3853                 vtxCoords[3 * thisVert + 0] += i * 2;
3854                 vtxCoords[3 * thisVert + 1] += j * 2;
3855                 vtxCoords[3 * thisVert + 2] += k * 2;
3856               }
3857             }
3858           }
3859         }
3860       }
3861       for (PetscInt i = 0; i < numBlocks; i++) {
3862         for (PetscInt f = 0; f < facesPerBlock; f++) {
3863           for (PetscInt v = 0; v < 4; v++) cells[i][f][v] = vertToTrueVert[cells[i][f][v]];
3864         }
3865       }
3866       PetscCall(PetscFree(vertToTrueVert));
3867       PetscCall(PetscFree(seen));
3868       cells_flat = cells[0][0];
3869       numEdges   = 0;
3870       for (PetscInt i = 0; i < numFaces; i++) {
3871         for (PetscInt e = 0; e < 4; e++) {
3872           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3873           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3874 
3875           for (PetscInt d = 0; d < 3; d++) {
3876             if (!periodic || periodic[0] != DM_BOUNDARY_PERIODIC) {
3877               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) numEdges++;
3878               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) numEdges++;
3879             }
3880           }
3881         }
3882       }
3883       PetscCall(PetscMalloc1(numEdges, &edges));
3884       PetscCall(PetscMalloc1(numEdges, &edgeSets));
3885       for (PetscInt edge = 0, i = 0; i < numFaces; i++) {
3886         for (PetscInt e = 0; e < 4; e++) {
3887           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3888           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3889 
3890           for (PetscInt d = 0; d < 3; d++) {
3891             if (!periodic || periodic[d] != DM_BOUNDARY_PERIODIC) {
3892               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) {
3893                 edges[edge][0]   = ev[0];
3894                 edges[edge][1]   = ev[1];
3895                 edgeSets[edge++] = 2 * d;
3896               }
3897               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) {
3898                 edges[edge][0]   = ev[0];
3899                 edges[edge][1]   = ev[1];
3900                 edgeSets[edge++] = 2 * d + 1;
3901               }
3902             }
3903           }
3904         }
3905       }
3906     }
3907     evalFunc   = TPSEvaluate_Gyroid;
3908     normalFunc = TPSExtrudeNormalFunc_Gyroid;
3909     break;
3910   }
3911 
3912   PetscCall(DMSetDimension(dm, topoDim));
3913   if (rank == 0) PetscCall(DMPlexBuildFromCellList(dm, numFaces, numVertices, 4, cells_flat));
3914   else PetscCall(DMPlexBuildFromCellList(dm, 0, 0, 0, NULL));
3915   PetscCall(PetscFree(cells_flat));
3916   {
3917     DM idm;
3918     PetscCall(DMPlexInterpolate(dm, &idm));
3919     PetscCall(DMPlexReplace_Internal(dm, &idm));
3920   }
3921   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, vtxCoords));
3922   else PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, NULL));
3923   PetscCall(PetscFree(vtxCoords));
3924 
3925   PetscCall(DMCreateLabel(dm, "Face Sets"));
3926   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3927   for (PetscInt e = 0; e < numEdges; e++) {
3928     PetscInt        njoin;
3929     const PetscInt *join, verts[] = {numFaces + edges[e][0], numFaces + edges[e][1]};
3930     PetscCall(DMPlexGetJoin(dm, 2, verts, &njoin, &join));
3931     PetscCheck(njoin == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected unique join of vertices %" PetscInt_FMT " and %" PetscInt_FMT, edges[e][0], edges[e][1]);
3932     PetscCall(DMLabelSetValue(label, join[0], edgeSets[e]));
3933     PetscCall(DMPlexRestoreJoin(dm, 2, verts, &njoin, &join));
3934   }
3935   PetscCall(PetscFree(edges));
3936   PetscCall(PetscFree(edgeSets));
3937   if (tps_distribute) {
3938     DM               pdm = NULL;
3939     PetscPartitioner part;
3940 
3941     PetscCall(DMPlexGetPartitioner(dm, &part));
3942     PetscCall(PetscPartitionerSetFromOptions(part));
3943     PetscCall(DMPlexDistribute(dm, 0, NULL, &pdm));
3944     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
3945     // Do not auto-distribute again
3946     PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
3947   }
3948 
3949   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3950   for (PetscInt refine = 0; refine < refinements; refine++) {
3951     PetscInt     m;
3952     DM           dmf;
3953     Vec          X;
3954     PetscScalar *x;
3955     PetscCall(DMRefine(dm, MPI_COMM_NULL, &dmf));
3956     PetscCall(DMPlexReplace_Internal(dm, &dmf));
3957 
3958     PetscCall(DMGetCoordinatesLocal(dm, &X));
3959     PetscCall(VecGetLocalSize(X, &m));
3960     PetscCall(VecGetArray(X, &x));
3961     for (PetscInt i = 0; i < m; i += 3) PetscCall(TPSNearestPoint(evalFunc, &x[i]));
3962     PetscCall(VecRestoreArray(X, &x));
3963   }
3964 
3965   // Face Sets has already been propagated to new vertices during refinement; this propagates to the initial vertices.
3966   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3967   PetscCall(DMPlexLabelComplete(dm, label));
3968 
3969   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
3970 
3971   if (thickness > 0) {
3972     DM              edm, cdm, ecdm;
3973     DMPlexTransform tr;
3974     const char     *prefix;
3975     PetscOptions    options;
3976     // Code from DMPlexExtrude
3977     PetscCall(DMPlexTransformCreate(PetscObjectComm((PetscObject)dm), &tr));
3978     PetscCall(DMPlexTransformSetDM(tr, dm));
3979     PetscCall(DMPlexTransformSetType(tr, DMPLEXEXTRUDE));
3980     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
3981     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)tr, prefix));
3982     PetscCall(PetscObjectGetOptions((PetscObject)dm, &options));
3983     PetscCall(PetscObjectSetOptions((PetscObject)tr, options));
3984     PetscCall(DMPlexTransformExtrudeSetLayers(tr, layers));
3985     PetscCall(DMPlexTransformExtrudeSetThickness(tr, thickness));
3986     PetscCall(DMPlexTransformExtrudeSetTensor(tr, PETSC_FALSE));
3987     PetscCall(DMPlexTransformExtrudeSetSymmetric(tr, PETSC_TRUE));
3988     PetscCall(DMPlexTransformExtrudeSetNormalFunction(tr, normalFunc));
3989     PetscCall(DMPlexTransformSetFromOptions(tr));
3990     PetscCall(PetscObjectSetOptions((PetscObject)tr, NULL));
3991     PetscCall(DMPlexTransformSetUp(tr));
3992     PetscCall(PetscObjectViewFromOptions((PetscObject)tr, NULL, "-dm_plex_tps_transform_view"));
3993     PetscCall(DMPlexTransformApply(tr, dm, &edm));
3994     PetscCall(DMCopyDisc(dm, edm));
3995     PetscCall(DMGetCoordinateDM(dm, &cdm));
3996     PetscCall(DMGetCoordinateDM(edm, &ecdm));
3997     PetscCall(DMCopyDisc(cdm, ecdm));
3998     PetscCall(DMPlexTransformCreateDiscLabels(tr, edm));
3999     PetscCall(DMPlexTransformDestroy(&tr));
4000     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, edm));
4001     PetscCall(DMPlexReplace_Internal(dm, &edm));
4002   }
4003   PetscFunctionReturn(PETSC_SUCCESS);
4004 }
4005 
4006 /*@
4007   DMPlexCreateTPSMesh - Create a distributed, interpolated mesh of a triply-periodic surface
4008 
4009   Collective
4010 
4011   Input Parameters:
4012 + comm           - The communicator for the `DM` object
4013 . tpstype        - Type of triply-periodic surface
4014 . extent         - Array of length 3 containing number of periods in each direction
4015 . periodic       - array of length 3 with periodicity, or `NULL` for non-periodic
4016 . tps_distribute - Distribute 2D manifold mesh prior to refinement and extrusion (more scalable)
4017 . refinements    - Number of factor-of-2 refinements of 2D manifold mesh
4018 . layers         - Number of cell layers extruded in normal direction
4019 - thickness      - Thickness in normal direction
4020 
4021   Output Parameter:
4022 . dm - The `DM` object
4023 
4024   Level: beginner
4025 
4026   Notes:
4027   This meshes the surface of the Schwarz P or Gyroid surfaces.  Schwarz P is the simplest member of the triply-periodic minimal surfaces.
4028   <https://en.wikipedia.org/wiki/Schwarz_minimal_surface#Schwarz_P_(%22Primitive%22)> and can be cut with "clean" boundaries.
4029   The Gyroid <https://en.wikipedia.org/wiki/Gyroid> is another triply-periodic minimal surface with applications in additive manufacturing; it is much more difficult to "cut" since there are no planes of symmetry.
4030   Our implementation creates a very coarse mesh of the surface and refines (by 4-way splitting) as many times as requested.
4031   On each refinement, all vertices are projected to their nearest point on the surface.
4032   This projection could readily be extended to related surfaces.
4033 
4034   See {cite}`maskery2018insights`
4035 
4036   The face (edge) sets for the Schwarz P surface are numbered $1(-x), 2(+x), 3(-y), 4(+y), 5(-z), 6(+z)$.
4037   When the mesh is refined, "Face Sets" contain the new vertices (created during refinement).
4038   Use `DMPlexLabelComplete()` to propagate to coarse-level vertices.
4039 
4040   Developer Notes:
4041   The Gyroid mesh does not currently mark boundary sets.
4042 
4043 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMSetType()`, `DMCreate()`
4044 @*/
4045 PetscErrorCode DMPlexCreateTPSMesh(MPI_Comm comm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness, DM *dm)
4046 {
4047   PetscFunctionBegin;
4048   PetscCall(DMCreate(comm, dm));
4049   PetscCall(DMSetType(*dm, DMPLEX));
4050   PetscCall(DMPlexCreateTPSMesh_Internal(*dm, tpstype, extent, periodic, tps_distribute, refinements, layers, thickness));
4051   PetscFunctionReturn(PETSC_SUCCESS);
4052 }
4053 
4054 /*@
4055   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
4056 
4057   Collective
4058 
4059   Input Parameters:
4060 + comm    - The communicator for the `DM` object
4061 . dim     - The dimension
4062 . simplex - Use simplices, or tensor product cells
4063 - R       - The radius
4064 
4065   Output Parameter:
4066 . dm - The `DM` object
4067 
4068   Level: beginner
4069 
4070 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBallMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
4071 @*/
4072 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscReal R, DM *dm)
4073 {
4074   PetscFunctionBegin;
4075   PetscAssertPointer(dm, 5);
4076   PetscCall(DMCreate(comm, dm));
4077   PetscCall(DMSetType(*dm, DMPLEX));
4078   PetscCall(DMPlexCreateSphereMesh_Internal(*dm, dim, simplex, R));
4079   PetscFunctionReturn(PETSC_SUCCESS);
4080 }
4081 
4082 static PetscErrorCode DMPlexCreateBallMesh_Internal(DM dm, PetscInt dim, PetscReal R)
4083 {
4084   DM          sdm, vol;
4085   DMLabel     bdlabel;
4086   const char *prefix;
4087 
4088   PetscFunctionBegin;
4089   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
4090   PetscCall(DMSetType(sdm, DMPLEX));
4091   PetscCall(DMGetOptionsPrefix(dm, &prefix));
4092   PetscCall(DMSetOptionsPrefix(sdm, prefix));
4093   PetscCall(DMAppendOptionsPrefix(sdm, "bd_"));
4094   PetscCall(DMPlexDistributeSetDefault(sdm, PETSC_FALSE));
4095   PetscCall(DMPlexCreateSphereMesh_Internal(sdm, dim - 1, PETSC_TRUE, R));
4096   PetscCall(DMSetFromOptions(sdm));
4097   PetscCall(DMViewFromOptions(sdm, NULL, "-dm_view"));
4098   PetscCall(DMPlexGenerate(sdm, NULL, PETSC_TRUE, &vol));
4099   PetscCall(DMDestroy(&sdm));
4100   PetscCall(DMPlexReplace_Internal(dm, &vol));
4101   PetscCall(DMCreateLabel(dm, "marker"));
4102   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
4103   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
4104   PetscCall(DMPlexLabelComplete(dm, bdlabel));
4105   PetscFunctionReturn(PETSC_SUCCESS);
4106 }
4107 
4108 /*@
4109   DMPlexCreateBallMesh - Creates a simplex mesh on the d-dimensional ball, B^d.
4110 
4111   Collective
4112 
4113   Input Parameters:
4114 + comm - The communicator for the `DM` object
4115 . dim  - The dimension
4116 - R    - The radius
4117 
4118   Output Parameter:
4119 . dm - The `DM` object
4120 
4121   Options Database Key:
4122 . bd_dm_refine - This will refine the surface mesh preserving the sphere geometry
4123 
4124   Level: beginner
4125 
4126 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
4127 @*/
4128 PetscErrorCode DMPlexCreateBallMesh(MPI_Comm comm, PetscInt dim, PetscReal R, DM *dm)
4129 {
4130   PetscFunctionBegin;
4131   PetscCall(DMCreate(comm, dm));
4132   PetscCall(DMSetType(*dm, DMPLEX));
4133   PetscCall(DMPlexCreateBallMesh_Internal(*dm, dim, R));
4134   PetscFunctionReturn(PETSC_SUCCESS);
4135 }
4136 
4137 static PetscErrorCode DMPlexCreateReferenceCell_Internal(DM rdm, DMPolytopeType ct)
4138 {
4139   PetscFunctionBegin;
4140   switch (ct) {
4141   case DM_POLYTOPE_POINT: {
4142     PetscInt    numPoints[1]        = {1};
4143     PetscInt    coneSize[1]         = {0};
4144     PetscInt    cones[1]            = {0};
4145     PetscInt    coneOrientations[1] = {0};
4146     PetscScalar vertexCoords[1]     = {0.0};
4147 
4148     PetscCall(DMSetDimension(rdm, 0));
4149     PetscCall(DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4150   } break;
4151   case DM_POLYTOPE_SEGMENT: {
4152     PetscInt    numPoints[2]        = {2, 1};
4153     PetscInt    coneSize[3]         = {2, 0, 0};
4154     PetscInt    cones[2]            = {1, 2};
4155     PetscInt    coneOrientations[2] = {0, 0};
4156     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
4157 
4158     PetscCall(DMSetDimension(rdm, 1));
4159     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4160   } break;
4161   case DM_POLYTOPE_POINT_PRISM_TENSOR: {
4162     PetscInt    numPoints[2]        = {2, 1};
4163     PetscInt    coneSize[3]         = {2, 0, 0};
4164     PetscInt    cones[2]            = {1, 2};
4165     PetscInt    coneOrientations[2] = {0, 0};
4166     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
4167 
4168     PetscCall(DMSetDimension(rdm, 1));
4169     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4170   } break;
4171   case DM_POLYTOPE_TRIANGLE: {
4172     PetscInt    numPoints[2]        = {3, 1};
4173     PetscInt    coneSize[4]         = {3, 0, 0, 0};
4174     PetscInt    cones[3]            = {1, 2, 3};
4175     PetscInt    coneOrientations[3] = {0, 0, 0};
4176     PetscScalar vertexCoords[6]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0};
4177 
4178     PetscCall(DMSetDimension(rdm, 2));
4179     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4180   } break;
4181   case DM_POLYTOPE_QUADRILATERAL: {
4182     PetscInt    numPoints[2]        = {4, 1};
4183     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
4184     PetscInt    cones[4]            = {1, 2, 3, 4};
4185     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
4186     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0};
4187 
4188     PetscCall(DMSetDimension(rdm, 2));
4189     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4190   } break;
4191   case DM_POLYTOPE_SEG_PRISM_TENSOR: {
4192     PetscInt    numPoints[2]        = {4, 1};
4193     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
4194     PetscInt    cones[4]            = {1, 2, 3, 4};
4195     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
4196     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0};
4197 
4198     PetscCall(DMSetDimension(rdm, 2));
4199     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4200   } break;
4201   case DM_POLYTOPE_TETRAHEDRON: {
4202     PetscInt    numPoints[2]        = {4, 1};
4203     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
4204     PetscInt    cones[4]            = {1, 2, 3, 4};
4205     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
4206     PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0};
4207 
4208     PetscCall(DMSetDimension(rdm, 3));
4209     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4210   } break;
4211   case DM_POLYTOPE_HEXAHEDRON: {
4212     PetscInt    numPoints[2]        = {8, 1};
4213     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
4214     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
4215     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
4216     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
4217 
4218     PetscCall(DMSetDimension(rdm, 3));
4219     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4220   } break;
4221   case DM_POLYTOPE_TRI_PRISM: {
4222     PetscInt    numPoints[2]        = {6, 1};
4223     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
4224     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
4225     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
4226     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
4227 
4228     PetscCall(DMSetDimension(rdm, 3));
4229     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4230   } break;
4231   case DM_POLYTOPE_TRI_PRISM_TENSOR: {
4232     PetscInt    numPoints[2]        = {6, 1};
4233     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
4234     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
4235     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
4236     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
4237 
4238     PetscCall(DMSetDimension(rdm, 3));
4239     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4240   } break;
4241   case DM_POLYTOPE_QUAD_PRISM_TENSOR: {
4242     PetscInt    numPoints[2]        = {8, 1};
4243     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
4244     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
4245     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
4246     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
4247 
4248     PetscCall(DMSetDimension(rdm, 3));
4249     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4250   } break;
4251   case DM_POLYTOPE_PYRAMID: {
4252     PetscInt    numPoints[2]        = {5, 1};
4253     PetscInt    coneSize[6]         = {5, 0, 0, 0, 0, 0};
4254     PetscInt    cones[5]            = {1, 2, 3, 4, 5};
4255     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
4256     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, 0.0, 0.0, 1.0};
4257 
4258     PetscCall(DMSetDimension(rdm, 3));
4259     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4260   } break;
4261   default:
4262     SETERRQ(PetscObjectComm((PetscObject)rdm), PETSC_ERR_ARG_WRONG, "Cannot create reference cell for cell type %s", DMPolytopeTypes[ct]);
4263   }
4264   {
4265     PetscInt Nv, v;
4266 
4267     /* Must create the celltype label here so that we do not automatically try to compute the types */
4268     PetscCall(DMCreateLabel(rdm, "celltype"));
4269     PetscCall(DMPlexSetCellType(rdm, 0, ct));
4270     PetscCall(DMPlexGetChart(rdm, NULL, &Nv));
4271     for (v = 1; v < Nv; ++v) PetscCall(DMPlexSetCellType(rdm, v, DM_POLYTOPE_POINT));
4272   }
4273   PetscCall(DMPlexInterpolateInPlace_Internal(rdm));
4274   PetscCall(PetscObjectSetName((PetscObject)rdm, DMPolytopeTypes[ct]));
4275   PetscFunctionReturn(PETSC_SUCCESS);
4276 }
4277 
4278 /*@
4279   DMPlexCreateReferenceCell - Create a `DMPLEX` with the appropriate FEM reference cell
4280 
4281   Collective
4282 
4283   Input Parameters:
4284 + comm - The communicator
4285 - ct   - The cell type of the reference cell
4286 
4287   Output Parameter:
4288 . refdm - The reference cell
4289 
4290   Level: intermediate
4291 
4292 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBoxMesh()`
4293 @*/
4294 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, DMPolytopeType ct, DM *refdm)
4295 {
4296   PetscFunctionBegin;
4297   PetscCall(DMCreate(comm, refdm));
4298   PetscCall(DMSetType(*refdm, DMPLEX));
4299   PetscCall(DMPlexCreateReferenceCell_Internal(*refdm, ct));
4300   PetscFunctionReturn(PETSC_SUCCESS);
4301 }
4302 
4303 static PetscErrorCode DMPlexCreateBoundaryLabel_Private(DM dm, const char name[])
4304 {
4305   DM        plex;
4306   DMLabel   label;
4307   PetscBool hasLabel;
4308 
4309   PetscFunctionBegin;
4310   PetscCall(DMHasLabel(dm, name, &hasLabel));
4311   if (hasLabel) PetscFunctionReturn(PETSC_SUCCESS);
4312   PetscCall(DMCreateLabel(dm, name));
4313   PetscCall(DMGetLabel(dm, name, &label));
4314   PetscCall(DMConvert(dm, DMPLEX, &plex));
4315   PetscCall(DMPlexMarkBoundaryFaces(plex, 1, label));
4316   PetscCall(DMPlexLabelComplete(plex, label));
4317   PetscCall(DMDestroy(&plex));
4318   PetscFunctionReturn(PETSC_SUCCESS);
4319 }
4320 
4321 /*
4322   We use the last coordinate as the radius, the inner radius is lower[dim-1] and the outer radius is upper[dim-1]. Then we map the first coordinate around the circle.
4323 
4324     (x, y) -> (r, theta) = (x[1], (x[0] - lower[0]) * 2\pi/(upper[0] - lower[0]))
4325 */
4326 static void boxToAnnulus(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
4327 {
4328   const PetscReal low = PetscRealPart(constants[0]);
4329   const PetscReal upp = PetscRealPart(constants[1]);
4330   const PetscReal r   = PetscRealPart(u[1]);
4331   const PetscReal th  = 2. * PETSC_PI * (PetscRealPart(u[0]) - low) / (upp - low);
4332 
4333   f0[0] = r * PetscCosReal(th);
4334   f0[1] = r * PetscSinReal(th);
4335 }
4336 
4337 // Insert vertices and their joins, marked by depth
4338 static PetscErrorCode ProcessCohesiveLabel_Vertices(DM dm, DMLabel label, DMLabel vlabel, PetscInt val, PetscInt n, const PetscInt vertices[])
4339 {
4340   PetscFunctionBegin;
4341   PetscCall(DMPlexMarkSubmesh_Interpolated(dm, vlabel, val, PETSC_FALSE, PETSC_FALSE, label, NULL));
4342   PetscFunctionReturn(PETSC_SUCCESS);
4343 }
4344 
4345 // Insert faces and their closures, marked by depth
4346 static PetscErrorCode ProcessCohesiveLabel_Faces(DM dm, DMLabel label, PetscInt n, const PetscInt faces[])
4347 {
4348   PetscFunctionBegin;
4349   for (PetscInt p = 0; p < n; ++p) {
4350     const PetscInt point   = faces[p];
4351     PetscInt      *closure = NULL;
4352     PetscInt       clSize, pdepth;
4353 
4354     PetscCall(DMPlexGetPointDepth(dm, point, &pdepth));
4355     PetscCall(DMLabelSetValue(label, point, pdepth));
4356     PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &clSize, &closure));
4357     for (PetscInt cl = 0; cl < clSize * 2; cl += 2) {
4358       PetscCall(DMPlexGetPointDepth(dm, closure[cl], &pdepth));
4359       PetscCall(DMLabelSetValue(label, closure[cl], pdepth));
4360     }
4361     PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &clSize, &closure));
4362   }
4363   PetscFunctionReturn(PETSC_SUCCESS);
4364 }
4365 
4366 PETSC_EXTERN PetscErrorCode PetscOptionsFindPairPrefix_Private(PetscOptions, const char pre[], const char name[], const char *option[], const char *value[], PetscBool *flg);
4367 
4368 const char *const DMPlexShapes[] = {"box", "box_surface", "ball", "sphere", "cylinder", "schwarz_p", "gyroid", "doublet", "annulus", "hypercubic", "zbox", "unknown", "DMPlexShape", "DM_SHAPE_", NULL};
4369 
4370 static PetscErrorCode DMPlexCreateFromOptions_Internal(PetscOptionItems *PetscOptionsObject, PetscBool *useCoordSpace, DM dm)
4371 {
4372   DMPlexShape    shape   = DM_SHAPE_BOX;
4373   DMPolytopeType cell    = DM_POLYTOPE_TRIANGLE;
4374   PetscInt       dim     = 2;
4375   PetscBool      simplex = PETSC_TRUE, interpolate = PETSC_TRUE, orient = PETSC_FALSE, adjCone = PETSC_FALSE, adjClosure = PETSC_TRUE, refDomain = PETSC_FALSE;
4376   PetscBool      flg, flg2, fflg, strflg, bdfflg, nameflg;
4377   MPI_Comm       comm;
4378   char           filename[PETSC_MAX_PATH_LEN]   = "<unspecified>";
4379   char           bdFilename[PETSC_MAX_PATH_LEN] = "<unspecified>";
4380   char           plexname[PETSC_MAX_PATH_LEN]   = "";
4381   const char    *option;
4382 
4383   PetscFunctionBegin;
4384   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromOptions, dm, 0, 0, 0));
4385   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
4386   /* TODO Turn this into a registration interface */
4387   PetscCall(PetscOptionsString("-dm_plex_filename", "File containing a mesh", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &fflg));
4388   PetscCall(PetscOptionsString("-dm_plex_file_contents", "Contents of a file format in a string", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &strflg));
4389   PetscCall(PetscOptionsString("-dm_plex_boundary_filename", "File containing a mesh boundary", "DMPlexCreateFromFile", bdFilename, bdFilename, sizeof(bdFilename), &bdfflg));
4390   PetscCall(PetscOptionsString("-dm_plex_name", "Name of the mesh in the file", "DMPlexCreateFromFile", plexname, plexname, sizeof(plexname), &nameflg));
4391   PetscCall(PetscOptionsEnum("-dm_plex_cell", "Cell shape", "", DMPolytopeTypes, (PetscEnum)cell, (PetscEnum *)&cell, NULL));
4392   PetscCall(PetscOptionsBool("-dm_plex_reference_cell_domain", "Use a reference cell domain", "", refDomain, &refDomain, NULL));
4393   PetscCall(PetscOptionsEnum("-dm_plex_shape", "Shape for built-in mesh", "", DMPlexShapes, (PetscEnum)shape, (PetscEnum *)&shape, &flg));
4394   PetscCall(PetscOptionsBoundedInt("-dm_plex_dim", "Topological dimension of the mesh", "DMGetDimension", dim, &dim, &flg, 0));
4395   PetscCall(PetscOptionsBool("-dm_plex_simplex", "Mesh cell shape", "", simplex, &simplex, &flg));
4396   PetscCall(PetscOptionsBool("-dm_plex_interpolate", "Flag to create edges and faces automatically", "", interpolate, &interpolate, &flg));
4397   PetscCall(PetscOptionsBool("-dm_plex_orient", "Orient the constructed mesh", "DMPlexOrient", orient, &orient, &flg));
4398   PetscCall(PetscOptionsBool("-dm_plex_adj_cone", "Set adjacency direction", "DMSetBasicAdjacency", adjCone, &adjCone, &flg));
4399   PetscCall(PetscOptionsBool("-dm_plex_adj_closure", "Set adjacency size", "DMSetBasicAdjacency", adjClosure, &adjClosure, &flg2));
4400   if (flg || flg2) PetscCall(DMSetBasicAdjacency(dm, adjCone, adjClosure));
4401   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_adj", "Debug output level all adjacency computations", "", 0, &((DM_Plex *)dm->data)->printAdj, NULL, 0));
4402 
4403   switch (cell) {
4404   case DM_POLYTOPE_POINT:
4405   case DM_POLYTOPE_SEGMENT:
4406   case DM_POLYTOPE_POINT_PRISM_TENSOR:
4407   case DM_POLYTOPE_TRIANGLE:
4408   case DM_POLYTOPE_QUADRILATERAL:
4409   case DM_POLYTOPE_TETRAHEDRON:
4410   case DM_POLYTOPE_HEXAHEDRON:
4411     *useCoordSpace = PETSC_TRUE;
4412     break;
4413   default:
4414     *useCoordSpace = PETSC_FALSE;
4415     break;
4416   }
4417 
4418   if (fflg) {
4419     DM          dmnew;
4420     const char *name;
4421 
4422     PetscCall(PetscObjectGetName((PetscObject)dm, &name));
4423     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), filename, nameflg ? plexname : name, interpolate, &dmnew));
4424     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4425     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4426   } else if (refDomain) {
4427     PetscCall(DMPlexCreateReferenceCell_Internal(dm, cell));
4428   } else if (bdfflg) {
4429     DM          bdm, dmnew;
4430     const char *name;
4431 
4432     PetscCall(PetscObjectGetName((PetscObject)dm, &name));
4433     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), bdFilename, nameflg ? plexname : name, interpolate, &bdm));
4434     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)bdm, "bd_"));
4435     PetscCall(DMSetFromOptions(bdm));
4436     PetscCall(DMPlexGenerate(bdm, NULL, interpolate, &dmnew));
4437     PetscCall(DMDestroy(&bdm));
4438     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4439     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4440   } else if (strflg) {
4441     DM          dmnew;
4442     PetscViewer viewer;
4443     const char *contents;
4444     char       *strname;
4445     char        tmpdir[PETSC_MAX_PATH_LEN];
4446     char        tmpfilename[PETSC_MAX_PATH_LEN];
4447     char        name[PETSC_MAX_PATH_LEN];
4448     MPI_Comm    comm;
4449     PetscMPIInt rank;
4450 
4451     PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
4452     PetscCallMPI(MPI_Comm_rank(comm, &rank));
4453     PetscCall(PetscStrchr(filename, ':', &strname));
4454     PetscCheck(strname, comm, PETSC_ERR_ARG_WRONG, "File contents must have the form \"ext:string_name\", not %s", filename);
4455     strname[0] = '\0';
4456     ++strname;
4457     PetscCall(PetscDLSym(NULL, strname, (void **)&contents));
4458     PetscCheck(contents, comm, PETSC_ERR_ARG_WRONG, "Could not locate mesh string %s", strname);
4459     PetscCall(PetscGetTmp(comm, tmpdir, PETSC_MAX_PATH_LEN));
4460     PetscCall(PetscStrlcat(tmpdir, "/meshXXXXXX", PETSC_MAX_PATH_LEN));
4461     PetscCall(PetscMkdtemp(tmpdir));
4462     PetscCall(PetscSNPrintf(tmpfilename, PETSC_MAX_PATH_LEN, "%s/mesh.%s", tmpdir, filename));
4463     PetscCall(PetscViewerASCIIOpen(comm, tmpfilename, &viewer));
4464     PetscCall(PetscViewerASCIIPrintf(viewer, "%s\n", contents));
4465     PetscCall(PetscViewerDestroy(&viewer));
4466     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), tmpfilename, plexname, interpolate, &dmnew));
4467     PetscCall(PetscRMTree(tmpdir));
4468     PetscCall(PetscSNPrintf(name, PETSC_MAX_PATH_LEN, "%s Mesh", strname));
4469     PetscCall(PetscObjectSetName((PetscObject)dm, name));
4470     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4471     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4472   } else {
4473     PetscCall(PetscObjectSetName((PetscObject)dm, DMPlexShapes[shape]));
4474     switch (shape) {
4475     case DM_SHAPE_BOX:
4476     case DM_SHAPE_ZBOX:
4477     case DM_SHAPE_ANNULUS: {
4478       PetscInt       faces[3]  = {0, 0, 0};
4479       PetscReal      lower[3]  = {0, 0, 0};
4480       PetscReal      upper[3]  = {1, 1, 1};
4481       DMBoundaryType bdt[3]    = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
4482       PetscBool      isAnnular = shape == DM_SHAPE_ANNULUS ? PETSC_TRUE : PETSC_FALSE;
4483       PetscInt       i, n;
4484 
4485       n = dim;
4486       for (i = 0; i < dim; ++i) faces[i] = (dim == 1 ? 1 : 4 - dim);
4487       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
4488       n = 3;
4489       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
4490       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4491       n = 3;
4492       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
4493       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4494       n = 3;
4495       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
4496       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4497 
4498       PetscCheck(!isAnnular || dim == 2, comm, PETSC_ERR_ARG_OUTOFRANGE, "Only two dimensional annuli have been implemented");
4499       if (isAnnular)
4500         for (i = 0; i < dim - 1; ++i) bdt[i] = DM_BOUNDARY_PERIODIC;
4501 
4502       switch (cell) {
4503       case DM_POLYTOPE_TRI_PRISM_TENSOR:
4504         PetscCall(DMPlexCreateWedgeBoxMesh_Internal(dm, faces, lower, upper, bdt));
4505         if (!interpolate) {
4506           DM udm;
4507 
4508           PetscCall(DMPlexUninterpolate(dm, &udm));
4509           PetscCall(DMPlexReplace_Internal(dm, &udm));
4510         }
4511         break;
4512       default:
4513         PetscCall(DMPlexCreateBoxMesh_Internal(dm, shape, dim, simplex, faces, lower, upper, bdt, interpolate));
4514         break;
4515       }
4516       if (isAnnular) {
4517         DM          cdm;
4518         PetscDS     cds;
4519         PetscScalar bounds[2] = {lower[0], upper[0]};
4520 
4521         // Fix coordinates for annular region
4522         PetscCall(DMSetPeriodicity(dm, NULL, NULL, NULL));
4523         PetscCall(DMSetCellCoordinatesLocal(dm, NULL));
4524         PetscCall(DMSetCellCoordinates(dm, NULL));
4525         PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_TRUE, NULL));
4526         PetscCall(DMGetCoordinateDM(dm, &cdm));
4527         PetscCall(DMGetDS(cdm, &cds));
4528         PetscCall(PetscDSSetConstants(cds, 2, bounds));
4529         PetscCall(DMPlexRemapGeometry(dm, 0.0, boxToAnnulus));
4530       }
4531     } break;
4532     case DM_SHAPE_BOX_SURFACE: {
4533       PetscInt  faces[3] = {0, 0, 0};
4534       PetscReal lower[3] = {0, 0, 0};
4535       PetscReal upper[3] = {1, 1, 1};
4536       PetscInt  i, n;
4537 
4538       n = dim + 1;
4539       for (i = 0; i < dim + 1; ++i) faces[i] = (dim + 1 == 1 ? 1 : 4 - (dim + 1));
4540       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
4541       n = 3;
4542       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
4543       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
4544       n = 3;
4545       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
4546       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
4547       PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(dm, dim + 1, faces, lower, upper, interpolate));
4548     } break;
4549     case DM_SHAPE_SPHERE: {
4550       PetscReal R = 1.0;
4551 
4552       PetscCall(PetscOptionsReal("-dm_plex_sphere_radius", "Radius of the sphere", "", R, &R, &flg));
4553       PetscCall(DMPlexCreateSphereMesh_Internal(dm, dim, simplex, R));
4554     } break;
4555     case DM_SHAPE_BALL: {
4556       PetscReal R = 1.0;
4557 
4558       PetscCall(PetscOptionsReal("-dm_plex_ball_radius", "Radius of the ball", "", R, &R, &flg));
4559       PetscCall(DMPlexCreateBallMesh_Internal(dm, dim, R));
4560     } break;
4561     case DM_SHAPE_CYLINDER: {
4562       DMBoundaryType bdt = DM_BOUNDARY_NONE;
4563       PetscInt       Nw  = 6;
4564       PetscInt       Nr  = 0;
4565 
4566       PetscCall(PetscOptionsEnum("-dm_plex_cylinder_bd", "Boundary type in the z direction", "", DMBoundaryTypes, (PetscEnum)bdt, (PetscEnum *)&bdt, NULL));
4567       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_wedges", "Number of wedges around the cylinder", "", Nw, &Nw, NULL));
4568       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_refine", "Number of refinements before projection", "", Nr, &Nr, NULL));
4569       switch (cell) {
4570       case DM_POLYTOPE_TRI_PRISM_TENSOR:
4571         PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(dm, Nw, interpolate));
4572         break;
4573       default:
4574         PetscCall(DMPlexCreateHexCylinderMesh_Internal(dm, bdt, Nr));
4575         break;
4576       }
4577     } break;
4578     case DM_SHAPE_SCHWARZ_P: // fallthrough
4579     case DM_SHAPE_GYROID: {
4580       PetscInt       extent[3] = {1, 1, 1}, refine = 0, layers = 0, three;
4581       PetscReal      thickness   = 0.;
4582       DMBoundaryType periodic[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
4583       DMPlexTPSType  tps_type    = shape == DM_SHAPE_SCHWARZ_P ? DMPLEX_TPS_SCHWARZ_P : DMPLEX_TPS_GYROID;
4584       PetscBool      tps_distribute;
4585       PetscCall(PetscOptionsIntArray("-dm_plex_tps_extent", "Number of replicas for each of three dimensions", NULL, extent, (three = 3, &three), NULL));
4586       PetscCall(PetscOptionsInt("-dm_plex_tps_refine", "Number of refinements", NULL, refine, &refine, NULL));
4587       PetscCall(PetscOptionsEnumArray("-dm_plex_tps_periodic", "Periodicity in each of three dimensions", NULL, DMBoundaryTypes, (PetscEnum *)periodic, (three = 3, &three), NULL));
4588       PetscCall(PetscOptionsInt("-dm_plex_tps_layers", "Number of layers in volumetric extrusion (or zero to not extrude)", NULL, layers, &layers, NULL));
4589       PetscCall(PetscOptionsReal("-dm_plex_tps_thickness", "Thickness of volumetric extrusion", NULL, thickness, &thickness, NULL));
4590       PetscCall(DMPlexDistributeGetDefault(dm, &tps_distribute));
4591       PetscCall(PetscOptionsBool("-dm_plex_tps_distribute", "Distribute the 2D mesh prior to refinement and extrusion", NULL, tps_distribute, &tps_distribute, NULL));
4592       PetscCall(DMPlexCreateTPSMesh_Internal(dm, tps_type, extent, periodic, tps_distribute, refine, layers, thickness));
4593     } break;
4594     case DM_SHAPE_DOUBLET: {
4595       DM        dmnew;
4596       PetscReal rl = 0.0;
4597 
4598       PetscCall(PetscOptionsReal("-dm_plex_doublet_refinementlimit", "Refinement limit", NULL, rl, &rl, NULL));
4599       PetscCall(DMPlexCreateDoublet(PetscObjectComm((PetscObject)dm), dim, simplex, interpolate, rl, &dmnew));
4600       PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4601       PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4602     } break;
4603     case DM_SHAPE_HYPERCUBIC: {
4604       PetscInt       *edges, overlap = 1;
4605       PetscReal      *lower, *upper;
4606       DMBoundaryType *bdt;
4607       PetscInt        n, d;
4608 
4609       *useCoordSpace = PETSC_FALSE;
4610       PetscCall(PetscMalloc4(dim, &edges, dim, &lower, dim, &upper, dim, &bdt));
4611       for (d = 0; d < dim; ++d) {
4612         edges[d] = 1;
4613         lower[d] = 0.;
4614         upper[d] = 1.;
4615         bdt[d]   = DM_BOUNDARY_PERIODIC;
4616       }
4617       n = dim;
4618       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", edges, &n, &flg));
4619       n = dim;
4620       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
4621       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4622       n = dim;
4623       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
4624       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4625       n = dim;
4626       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
4627       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4628       PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", overlap, &overlap, NULL, 0));
4629       PetscCall(DMPlexCreateHypercubicMesh_Internal(dm, dim, lower, upper, edges, overlap, bdt));
4630       PetscCall(PetscFree4(edges, lower, upper, bdt));
4631     } break;
4632     default:
4633       SETERRQ(comm, PETSC_ERR_SUP, "Domain shape %s is unsupported", DMPlexShapes[shape]);
4634     }
4635   }
4636   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
4637   if (!((PetscObject)dm)->name && nameflg) PetscCall(PetscObjectSetName((PetscObject)dm, plexname));
4638   if (orient) PetscCall(DMPlexOrient(dm));
4639   // Allow label creation
4640   PetscCall(PetscOptionsFindPairPrefix_Private(NULL, ((PetscObject)dm)->prefix, "-dm_plex_label_", &option, NULL, &flg));
4641   if (flg) {
4642     DMLabel     label;
4643     PetscInt    points[1024], n = 1024;
4644     char        fulloption[PETSC_MAX_PATH_LEN];
4645     const char *name = &option[14];
4646 
4647     PetscCall(DMCreateLabel(dm, name));
4648     PetscCall(DMGetLabel(dm, name, &label));
4649     fulloption[0] = '-';
4650     fulloption[1] = 0;
4651     PetscCall(PetscStrlcat(fulloption, option, PETSC_MAX_PATH_LEN));
4652     PetscCall(PetscOptionsGetIntArray(NULL, ((PetscObject)dm)->prefix, fulloption, points, &n, NULL));
4653     for (PetscInt p = 0; p < n; ++p) PetscCall(DMLabelSetValue(label, points[p], 1));
4654   }
4655   // Allow cohesive label creation
4656   //   Faces are input, completed, and all points are marked with their depth
4657   PetscCall(PetscOptionsFindPairPrefix_Private(NULL, ((PetscObject)dm)->prefix, "-dm_plex_cohesive_label_", &option, NULL, &flg));
4658   if (flg) {
4659     DMLabel   label;
4660     PetscInt  points[1024], n, pStart, pEnd, Nl = 1;
4661     PetscBool noCreate = PETSC_FALSE;
4662     char      fulloption[PETSC_MAX_PATH_LEN];
4663     char      name[PETSC_MAX_PATH_LEN];
4664     size_t    len;
4665 
4666     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
4667     PetscCall(PetscStrncpy(name, &option[23], PETSC_MAX_PATH_LEN));
4668     PetscCall(PetscStrlen(name, &len));
4669     if (name[len - 1] == '0') Nl = 10;
4670     for (PetscInt l = 0; l < Nl; ++l) {
4671       if (l > 0) name[len - 1] = (char)('0' + l);
4672       fulloption[0] = 0;
4673       PetscCall(PetscStrlcat(fulloption, "-dm_plex_cohesive_label_", 32));
4674       PetscCall(PetscStrlcat(fulloption, name, PETSC_MAX_PATH_LEN - 32));
4675       n = 1024;
4676       PetscCall(PetscOptionsGetIntArray(NULL, ((PetscObject)dm)->prefix, fulloption, points, &n, &flg));
4677       if (!flg) break;
4678       PetscCall(DMHasLabel(dm, name, &noCreate));
4679       if (noCreate) {
4680         DMLabel         inlabel;
4681         IS              pointIS;
4682         const PetscInt *lpoints;
4683         PetscInt        pdep, ln, inval = points[0];
4684         char            newname[PETSC_MAX_PATH_LEN];
4685 
4686         PetscCheck(n == 1, comm, PETSC_ERR_ARG_WRONG, "Must specify a label value with this option");
4687         PetscCall(DMGetLabel(dm, name, &inlabel));
4688         PetscCall(DMLabelGetStratumIS(inlabel, inval, &pointIS));
4689         PetscCall(ISGetLocalSize(pointIS, &ln));
4690         PetscCall(ISGetIndices(pointIS, &lpoints));
4691         PetscCall(DMPlexGetPointDepth(dm, lpoints[0], &pdep));
4692         PetscCall(PetscSNPrintf(newname, PETSC_MAX_PATH_LEN, "%s%" PetscInt_FMT, name, points[0]));
4693         PetscCall(DMCreateLabel(dm, newname));
4694         PetscCall(DMGetLabel(dm, newname, &label));
4695         if (!pdep) PetscCall(ProcessCohesiveLabel_Vertices(dm, label, inlabel, inval, ln, lpoints));
4696         else PetscCall(ProcessCohesiveLabel_Faces(dm, label, ln, lpoints));
4697         PetscCall(ISRestoreIndices(pointIS, &lpoints));
4698         PetscCall(ISDestroy(&pointIS));
4699       } else {
4700         PetscCall(DMCreateLabel(dm, name));
4701         PetscCall(DMGetLabel(dm, name, &label));
4702         if (pStart >= pEnd) n = 0;
4703         PetscCall(ProcessCohesiveLabel_Faces(dm, label, n, points));
4704       }
4705       PetscCall(DMPlexOrientLabel(dm, label));
4706       PetscCall(DMPlexLabelCohesiveComplete(dm, label, NULL, 1, PETSC_FALSE, PETSC_FALSE, NULL));
4707     }
4708   }
4709   PetscCall(DMViewFromOptions(dm, NULL, "-created_dm_view"));
4710   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromOptions, dm, 0, 0, 0));
4711   PetscFunctionReturn(PETSC_SUCCESS);
4712 }
4713 
4714 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
4715 {
4716   DM_Plex  *mesh = (DM_Plex *)dm->data;
4717   PetscBool flg, flg2;
4718   char      bdLabel[PETSC_MAX_PATH_LEN];
4719   char      method[PETSC_MAX_PATH_LEN];
4720 
4721   PetscFunctionBegin;
4722   /* Handle viewing */
4723   PetscCall(PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL));
4724   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fem", "Debug output level for all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL, 0));
4725   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fvm", "Debug output level for all fvm computations", "DMPlexSNESComputeResidualFVM", 0, &mesh->printFVM, NULL, 0));
4726   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_adj", "Debug output level all adjacency computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printAdj, NULL, 0));
4727   PetscCall(PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL));
4728   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL, 0));
4729   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_locate", "Debug output level all point location computations", "DMLocatePoints", 0, &mesh->printLocate, NULL, 0));
4730   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_project", "Debug output level all projection computations", "DMPlexProject", 0, &mesh->printProject, NULL, 0));
4731   PetscCall(DMMonitorSetFromOptions(dm, "-dm_plex_monitor_throughput", "Monitor the simulation throughput", "DMPlexMonitorThroughput", DMPlexMonitorThroughput, NULL, &flg));
4732   if (flg) PetscCall(PetscLogDefaultBegin());
4733   /* Labeling */
4734   PetscCall(PetscOptionsString("-dm_plex_boundary_label", "Label to mark the mesh boundary", "", bdLabel, bdLabel, sizeof(bdLabel), &flg));
4735   if (flg) PetscCall(DMPlexCreateBoundaryLabel_Private(dm, bdLabel));
4736   /* Point Location */
4737   PetscCall(PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL));
4738   /* Partitioning and distribution */
4739   PetscCall(PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL));
4740   /* Reordering */
4741   PetscCall(PetscOptionsBool("-dm_reorder_section", "Compute point permutation for local section", "DMReorderSectionSetDefault", PETSC_FALSE, &flg2, &flg));
4742   if (flg) PetscCall(DMReorderSectionSetDefault(dm, flg2 ? DM_REORDER_DEFAULT_TRUE : DM_REORDER_DEFAULT_FALSE));
4743   PetscCall(PetscOptionsString("-dm_reorder_section_type", "Reordering method for local section", "DMReorderSectionSetType", method, method, PETSC_MAX_PATH_LEN, &flg));
4744   if (flg) PetscCall(DMReorderSectionSetType(dm, method));
4745   /* Generation and remeshing */
4746   PetscCall(PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL));
4747   /* Projection behavior */
4748   PetscCall(PetscOptionsBoundedInt("-dm_plex_max_projection_height", "Maximum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL, 0));
4749   PetscCall(PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL));
4750   /* Checking structure */
4751   {
4752     PetscBool all = PETSC_FALSE;
4753 
4754     PetscCall(PetscOptionsBool("-dm_plex_check_all", "Perform all basic checks", "DMPlexCheck", PETSC_FALSE, &all, NULL));
4755     if (all) {
4756       PetscCall(DMPlexCheck(dm));
4757     } else {
4758       PetscCall(PetscOptionsBool("-dm_plex_check_symmetry", "Check that the adjacency information in the mesh is symmetric", "DMPlexCheckSymmetry", PETSC_FALSE, &flg, &flg2));
4759       if (flg && flg2) PetscCall(DMPlexCheckSymmetry(dm));
4760       PetscCall(PetscOptionsBool("-dm_plex_check_skeleton", "Check that each cell has the correct number of vertices (only for homogeneous simplex or tensor meshes)", "DMPlexCheckSkeleton", PETSC_FALSE, &flg, &flg2));
4761       if (flg && flg2) PetscCall(DMPlexCheckSkeleton(dm, 0));
4762       PetscCall(PetscOptionsBool("-dm_plex_check_faces", "Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type", "DMPlexCheckFaces", PETSC_FALSE, &flg, &flg2));
4763       if (flg && flg2) PetscCall(DMPlexCheckFaces(dm, 0));
4764       PetscCall(PetscOptionsBool("-dm_plex_check_geometry", "Check that cells have positive volume", "DMPlexCheckGeometry", PETSC_FALSE, &flg, &flg2));
4765       if (flg && flg2) PetscCall(DMPlexCheckGeometry(dm));
4766       PetscCall(PetscOptionsBool("-dm_plex_check_pointsf", "Check some necessary conditions for PointSF", "DMPlexCheckPointSF", PETSC_FALSE, &flg, &flg2));
4767       if (flg && flg2) PetscCall(DMPlexCheckPointSF(dm, NULL, PETSC_FALSE));
4768       PetscCall(PetscOptionsBool("-dm_plex_check_interface_cones", "Check points on inter-partition interfaces have conforming order of cone points", "DMPlexCheckInterfaceCones", PETSC_FALSE, &flg, &flg2));
4769       if (flg && flg2) PetscCall(DMPlexCheckInterfaceCones(dm));
4770     }
4771     PetscCall(PetscOptionsBool("-dm_plex_check_cell_shape", "Check cell shape", "DMPlexCheckCellShape", PETSC_FALSE, &flg, &flg2));
4772     if (flg && flg2) PetscCall(DMPlexCheckCellShape(dm, PETSC_TRUE, PETSC_DETERMINE));
4773   }
4774   {
4775     PetscReal scale = 1.0;
4776 
4777     PetscCall(PetscOptionsReal("-dm_plex_scale", "Scale factor for mesh coordinates", "DMPlexScale", scale, &scale, &flg));
4778     if (flg) {
4779       Vec coordinates, coordinatesLocal;
4780 
4781       PetscCall(DMGetCoordinates(dm, &coordinates));
4782       PetscCall(DMGetCoordinatesLocal(dm, &coordinatesLocal));
4783       PetscCall(VecScale(coordinates, scale));
4784       PetscCall(VecScale(coordinatesLocal, scale));
4785     }
4786   }
4787   PetscCall(PetscPartitionerSetFromOptions(mesh->partitioner));
4788   PetscFunctionReturn(PETSC_SUCCESS);
4789 }
4790 
4791 PetscErrorCode DMSetFromOptions_Overlap_Plex(DM dm, PetscOptionItems *PetscOptionsObject, PetscInt *overlap)
4792 {
4793   PetscInt  numOvLabels = 16, numOvExLabels = 16;
4794   char     *ovLabelNames[16], *ovExLabelNames[16];
4795   PetscInt  numOvValues = 16, numOvExValues = 16, l;
4796   PetscBool flg;
4797 
4798   PetscFunctionBegin;
4799   PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", *overlap, overlap, NULL, 0));
4800   PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_labels", "List of overlap label names", "DMPlexDistribute", ovLabelNames, &numOvLabels, &flg));
4801   if (!flg) numOvLabels = 0;
4802   if (numOvLabels) {
4803     ((DM_Plex *)dm->data)->numOvLabels = numOvLabels;
4804     for (l = 0; l < numOvLabels; ++l) {
4805       PetscCall(DMGetLabel(dm, ovLabelNames[l], &((DM_Plex *)dm->data)->ovLabels[l]));
4806       PetscCheck(((DM_Plex *)dm->data)->ovLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovLabelNames[l]);
4807       PetscCall(PetscFree(ovLabelNames[l]));
4808     }
4809     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_values", "List of overlap label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovValues, &numOvValues, &flg));
4810     if (!flg) numOvValues = 0;
4811     PetscCheck(numOvLabels == numOvValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvLabels, numOvValues);
4812 
4813     PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_exclude_labels", "List of overlap exclude label names", "DMPlexDistribute", ovExLabelNames, &numOvExLabels, &flg));
4814     if (!flg) numOvExLabels = 0;
4815     ((DM_Plex *)dm->data)->numOvExLabels = numOvExLabels;
4816     for (l = 0; l < numOvExLabels; ++l) {
4817       PetscCall(DMGetLabel(dm, ovExLabelNames[l], &((DM_Plex *)dm->data)->ovExLabels[l]));
4818       PetscCheck(((DM_Plex *)dm->data)->ovExLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovExLabelNames[l]);
4819       PetscCall(PetscFree(ovExLabelNames[l]));
4820     }
4821     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_exclude_values", "List of overlap exclude label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovExValues, &numOvExValues, &flg));
4822     if (!flg) numOvExValues = 0;
4823     PetscCheck(numOvExLabels == numOvExValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of exclude labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvExLabels, numOvExValues);
4824   }
4825   PetscFunctionReturn(PETSC_SUCCESS);
4826 }
4827 
4828 static PetscErrorCode DMSetFromOptions_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
4829 {
4830   PetscFunctionList    ordlist;
4831   char                 oname[256];
4832   char                 sublabelname[PETSC_MAX_PATH_LEN] = "";
4833   DMReorderDefaultFlag reorder;
4834   PetscReal            volume    = -1.0;
4835   PetscInt             prerefine = 0, refine = 0, r, coarsen = 0, overlap = 0, extLayers = 0, dim;
4836   PetscBool            uniformOrig = PETSC_FALSE, created = PETSC_FALSE, uniform = PETSC_TRUE, distribute, saveSF = PETSC_FALSE, interpolate = PETSC_TRUE, coordSpace = PETSC_TRUE, remap = PETSC_TRUE, ghostCells = PETSC_FALSE, isHierarchy, flg;
4837 
4838   PetscFunctionBegin;
4839   PetscOptionsHeadBegin(PetscOptionsObject, "DMPlex Options");
4840   if (dm->cloneOpts) goto non_refine;
4841   /* Handle automatic creation */
4842   PetscCall(DMGetDimension(dm, &dim));
4843   if (dim < 0) {
4844     PetscCall(DMPlexCreateFromOptions_Internal(PetscOptionsObject, &coordSpace, dm));
4845     created = PETSC_TRUE;
4846   }
4847   PetscCall(DMGetDimension(dm, &dim));
4848   /* Handle interpolation before distribution */
4849   PetscCall(PetscOptionsBool("-dm_plex_interpolate_pre", "Flag to interpolate mesh before distribution", "", interpolate, &interpolate, &flg));
4850   if (flg) {
4851     DMPlexInterpolatedFlag interpolated;
4852 
4853     PetscCall(DMPlexIsInterpolated(dm, &interpolated));
4854     if (interpolated == DMPLEX_INTERPOLATED_FULL && !interpolate) {
4855       DM udm;
4856 
4857       PetscCall(DMPlexUninterpolate(dm, &udm));
4858       PetscCall(DMPlexReplace_Internal(dm, &udm));
4859     } else if (interpolated != DMPLEX_INTERPOLATED_FULL && interpolate) {
4860       DM idm;
4861 
4862       PetscCall(DMPlexInterpolate(dm, &idm));
4863       PetscCall(DMPlexReplace_Internal(dm, &idm));
4864     }
4865   }
4866   // Handle submesh selection before distribution
4867   PetscCall(PetscOptionsString("-dm_plex_submesh", "Label to use for submesh selection", "", sublabelname, sublabelname, PETSC_MAX_PATH_LEN, &flg));
4868   if (flg) {
4869     DM              subdm;
4870     DMLabel         label;
4871     IS              valueIS, pointIS;
4872     const PetscInt *values, *points;
4873     PetscBool       markedFaces = PETSC_FALSE;
4874     PetscInt        Nv, value, Np;
4875 
4876     PetscCall(DMGetLabel(dm, sublabelname, &label));
4877     PetscCall(DMLabelGetNumValues(label, &Nv));
4878     PetscCheck(Nv == 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Only a single label value is currently supported for submesh selection, not %" PetscInt_FMT, Nv);
4879     PetscCall(DMLabelGetValueIS(label, &valueIS));
4880     PetscCall(ISGetIndices(valueIS, &values));
4881     value = values[0];
4882     PetscCall(ISRestoreIndices(valueIS, &values));
4883     PetscCall(ISDestroy(&valueIS));
4884     PetscCall(DMLabelGetStratumSize(label, value, &Np));
4885     PetscCall(DMLabelGetStratumIS(label, value, &pointIS));
4886     PetscCall(ISGetIndices(pointIS, &points));
4887     for (PetscInt p = 0; p < Np; ++p) {
4888       PetscInt pdepth;
4889 
4890       PetscCall(DMPlexGetPointDepth(dm, points[p], &pdepth));
4891       if (pdepth) {
4892         markedFaces = PETSC_TRUE;
4893         break;
4894       }
4895     }
4896     PetscCall(ISRestoreIndices(pointIS, &points));
4897     PetscCall(ISDestroy(&pointIS));
4898     PetscCall(DMPlexCreateSubmesh(dm, label, value, markedFaces, &subdm));
4899     PetscCall(DMPlexReplace_Internal(dm, &subdm));
4900     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4901   }
4902   /* Handle DMPlex refinement before distribution */
4903   PetscCall(DMPlexGetRefinementUniform(dm, &uniformOrig));
4904   PetscCall(PetscOptionsBoundedInt("-dm_refine_pre", "The number of refinements before distribution", "DMCreate", prerefine, &prerefine, NULL, 0));
4905   PetscCall(PetscOptionsBool("-dm_refine_remap_pre", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
4906   PetscCall(PetscOptionsBool("-dm_refine_uniform_pre", "Flag for uniform refinement before distribution", "DMCreate", uniform, &uniform, &flg));
4907   if (flg) PetscCall(DMPlexSetRefinementUniform(dm, uniform));
4908   PetscCall(PetscOptionsReal("-dm_refine_volume_limit_pre", "The maximum cell volume after refinement before distribution", "DMCreate", volume, &volume, &flg));
4909   if (flg) {
4910     PetscCall(DMPlexSetRefinementUniform(dm, PETSC_FALSE));
4911     PetscCall(DMPlexSetRefinementLimit(dm, volume));
4912     prerefine = PetscMax(prerefine, 1);
4913   }
4914   if (prerefine) PetscCall(DMLocalizeCoordinates(dm));
4915   for (r = 0; r < prerefine; ++r) {
4916     DM             rdm;
4917     PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4918 
4919     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4920     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
4921     PetscCall(DMPlexReplace_Internal(dm, &rdm));
4922     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4923     if (coordFunc && remap) {
4924       PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4925       ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4926     }
4927   }
4928   PetscCall(DMPlexSetRefinementUniform(dm, uniformOrig));
4929   /* Handle DMPlex extrusion before distribution */
4930   PetscCall(PetscOptionsBoundedInt("-dm_extrude", "The number of layers to extrude", "", extLayers, &extLayers, NULL, 0));
4931   if (extLayers) {
4932     DM edm;
4933 
4934     PetscCall(DMExtrude(dm, extLayers, &edm));
4935     PetscCall(DMPlexReplace_Internal(dm, &edm));
4936     ((DM_Plex *)dm->data)->coordFunc = NULL;
4937     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4938     extLayers = 0;
4939     PetscCall(DMGetDimension(dm, &dim));
4940   }
4941   /* Handle DMPlex reordering before distribution */
4942   PetscCall(DMPlexReorderGetDefault(dm, &reorder));
4943   PetscCall(MatGetOrderingList(&ordlist));
4944   PetscCall(PetscStrncpy(oname, MATORDERINGNATURAL, sizeof(oname)));
4945   PetscCall(PetscOptionsFList("-dm_plex_reorder", "Set mesh reordering type", "DMPlexGetOrdering", ordlist, MATORDERINGNATURAL, oname, sizeof(oname), &flg));
4946   if (reorder == DM_REORDER_DEFAULT_TRUE || flg) {
4947     DM pdm;
4948     IS perm;
4949 
4950     PetscCall(DMPlexGetOrdering(dm, oname, NULL, &perm));
4951     PetscCall(DMPlexPermute(dm, perm, &pdm));
4952     PetscCall(ISDestroy(&perm));
4953     PetscCall(DMPlexReplace_Internal(dm, &pdm));
4954     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4955   }
4956   /* Handle DMPlex distribution */
4957   PetscCall(DMPlexDistributeGetDefault(dm, &distribute));
4958   PetscCall(PetscOptionsBool("-dm_distribute", "Flag to redistribute a mesh among processes", "DMPlexDistribute", distribute, &distribute, NULL));
4959   PetscCall(PetscOptionsBool("-dm_distribute_save_sf", "Flag to save the migration SF", "DMPlexSetMigrationSF", saveSF, &saveSF, NULL));
4960   PetscCall(DMSetFromOptions_Overlap_Plex(dm, PetscOptionsObject, &overlap));
4961   if (distribute) {
4962     DM               pdm = NULL;
4963     PetscPartitioner part;
4964     PetscSF          sfMigration;
4965 
4966     PetscCall(DMPlexGetPartitioner(dm, &part));
4967     PetscCall(PetscPartitionerSetFromOptions(part));
4968     PetscCall(DMPlexDistribute(dm, overlap, &sfMigration, &pdm));
4969     if (pdm) {
4970       // Delete the local section to force the existing one to be rebuilt with the distributed DM
4971       PetscCall(DMSetLocalSection(dm, pdm->localSection));
4972       PetscCall(DMPlexReplace_Internal(dm, &pdm));
4973     }
4974     if (saveSF) PetscCall(DMPlexSetMigrationSF(dm, sfMigration));
4975     PetscCall(PetscSFDestroy(&sfMigration));
4976   }
4977 
4978   {
4979     PetscBool useBoxLabel = PETSC_FALSE;
4980     PetscCall(PetscOptionsBool("-dm_plex_box_label", "Create 'Face Sets' assuming boundary faces align with cartesian directions", "DMCreate", useBoxLabel, &useBoxLabel, NULL));
4981     if (useBoxLabel) {
4982       PetscInt       n      = 3;
4983       DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
4984 
4985       PetscCall(PetscOptionsEnumArray("-dm_plex_box_label_bd", "Boundary type for each dimension when using -dm_plex_box_label", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
4986       PetscCheck(!flg || !(n != dim), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4987       PetscCall(DMPlexSetBoxLabel_Internal(dm, bdt));
4988     }
4989   }
4990   /* Must check CEED options before creating function space for coordinates */
4991   {
4992     PetscBool useCeed = PETSC_FALSE, flg;
4993 
4994     PetscCall(PetscOptionsBool("-dm_plex_use_ceed", "Use LibCEED as the FEM backend", "DMPlexSetUseCeed", useCeed, &useCeed, &flg));
4995     if (flg) PetscCall(DMPlexSetUseCeed(dm, useCeed));
4996   }
4997   /* Create coordinate space */
4998   if (created) {
4999     DM_Plex  *mesh   = (DM_Plex *)dm->data;
5000     PetscInt  degree = 1, deg;
5001     PetscInt  height = 0;
5002     DM        cdm;
5003     PetscBool flg, localize = PETSC_TRUE, sparseLocalize = PETSC_TRUE;
5004 
5005     PetscCall(PetscOptionsBool("-dm_coord_space", "Use an FEM space for coordinates", "", coordSpace, &coordSpace, &flg));
5006     PetscCall(PetscOptionsInt("-dm_coord_petscspace_degree", "FEM degree for coordinate space", "", degree, &degree, NULL));
5007     PetscCall(DMGetCoordinateDegree_Internal(dm, &deg));
5008     if (coordSpace && deg <= 1) PetscCall(DMPlexCreateCoordinateSpace(dm, degree, PETSC_TRUE, mesh->coordFunc));
5009     PetscCall(DMGetCoordinateDM(dm, &cdm));
5010     if (flg && !coordSpace) {
5011       PetscDS      cds;
5012       PetscObject  obj;
5013       PetscClassId id;
5014 
5015       PetscCall(DMGetDS(cdm, &cds));
5016       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
5017       PetscCall(PetscObjectGetClassId(obj, &id));
5018       if (id == PETSCFE_CLASSID) {
5019         PetscContainer dummy;
5020 
5021         PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &dummy));
5022         PetscCall(PetscObjectSetName((PetscObject)dummy, "coordinates"));
5023         PetscCall(DMSetField(cdm, 0, NULL, (PetscObject)dummy));
5024         PetscCall(PetscContainerDestroy(&dummy));
5025         PetscCall(DMClearDS(cdm));
5026       }
5027       mesh->coordFunc = NULL;
5028     }
5029     PetscCall(PetscOptionsBool("-dm_localize", "Localize mesh coordinates", "", localize, &localize, NULL));
5030     PetscCall(PetscOptionsBool("-dm_sparse_localize", "Localize only necessary cells", "DMSetSparseLocalize", sparseLocalize, &sparseLocalize, &flg));
5031     if (flg) PetscCall(DMSetSparseLocalize(dm, sparseLocalize));
5032     PetscCall(PetscOptionsInt("-dm_localize_height", "Localize edges and faces in addition to cells", "", height, &height, &flg));
5033     if (flg) PetscCall(DMPlexSetMaxProjectionHeight(cdm, height));
5034     if (localize) PetscCall(DMLocalizeCoordinates(dm));
5035   }
5036   /* Handle DMPlex refinement */
5037   remap = PETSC_TRUE;
5038   PetscCall(PetscOptionsBoundedInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL, 0));
5039   PetscCall(PetscOptionsBool("-dm_refine_remap", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
5040   PetscCall(PetscOptionsBoundedInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy, 0));
5041   if (refine) PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
5042   if (refine && isHierarchy) {
5043     DM *dms, coarseDM;
5044 
5045     PetscCall(DMGetCoarseDM(dm, &coarseDM));
5046     PetscCall(PetscObjectReference((PetscObject)coarseDM));
5047     PetscCall(PetscMalloc1(refine, &dms));
5048     PetscCall(DMRefineHierarchy(dm, refine, dms));
5049     /* Total hack since we do not pass in a pointer */
5050     PetscCall(DMPlexSwap_Static(dm, dms[refine - 1]));
5051     if (refine == 1) {
5052       PetscCall(DMSetCoarseDM(dm, dms[0]));
5053       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
5054     } else {
5055       PetscCall(DMSetCoarseDM(dm, dms[refine - 2]));
5056       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
5057       PetscCall(DMSetCoarseDM(dms[0], dms[refine - 1]));
5058       PetscCall(DMPlexSetRegularRefinement(dms[0], PETSC_TRUE));
5059     }
5060     PetscCall(DMSetCoarseDM(dms[refine - 1], coarseDM));
5061     PetscCall(PetscObjectDereference((PetscObject)coarseDM));
5062     /* Free DMs */
5063     for (r = 0; r < refine; ++r) {
5064       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
5065       PetscCall(DMDestroy(&dms[r]));
5066     }
5067     PetscCall(PetscFree(dms));
5068   } else {
5069     for (r = 0; r < refine; ++r) {
5070       DM             rdm;
5071       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
5072 
5073       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5074       PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
5075       /* Total hack since we do not pass in a pointer */
5076       PetscCall(DMPlexReplace_Internal(dm, &rdm));
5077       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5078       if (coordFunc && remap) {
5079         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
5080         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
5081       }
5082     }
5083   }
5084   /* Handle DMPlex coarsening */
5085   PetscCall(PetscOptionsBoundedInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL, 0));
5086   PetscCall(PetscOptionsBoundedInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy, 0));
5087   if (coarsen && isHierarchy) {
5088     DM *dms;
5089 
5090     PetscCall(PetscMalloc1(coarsen, &dms));
5091     PetscCall(DMCoarsenHierarchy(dm, coarsen, dms));
5092     /* Free DMs */
5093     for (r = 0; r < coarsen; ++r) {
5094       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
5095       PetscCall(DMDestroy(&dms[r]));
5096     }
5097     PetscCall(PetscFree(dms));
5098   } else {
5099     for (r = 0; r < coarsen; ++r) {
5100       DM             cdm;
5101       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
5102 
5103       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5104       PetscCall(DMCoarsen(dm, PetscObjectComm((PetscObject)dm), &cdm));
5105       /* Total hack since we do not pass in a pointer */
5106       PetscCall(DMPlexReplace_Internal(dm, &cdm));
5107       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5108       if (coordFunc) {
5109         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
5110         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
5111       }
5112     }
5113   }
5114   // Handle coordinate remapping
5115   remap = PETSC_FALSE;
5116   PetscCall(PetscOptionsBool("-dm_coord_remap", "Flag to control coordinate remapping", "", remap, &remap, NULL));
5117   if (remap) {
5118     DMPlexCoordMap map     = DM_COORD_MAP_NONE;
5119     PetscPointFunc mapFunc = NULL;
5120     PetscScalar    params[16];
5121     PetscInt       Np = PETSC_STATIC_ARRAY_LENGTH(params), cdim;
5122     MPI_Comm       comm;
5123 
5124     PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
5125     PetscCall(DMGetCoordinateDim(dm, &cdim));
5126     PetscCall(PetscOptionsScalarArray("-dm_coord_map_params", "Parameters for the coordinate remapping", "", params, &Np, &flg));
5127     if (!flg) Np = 0;
5128     // TODO Allow user to pass a map function by name
5129     PetscCall(PetscOptionsEnum("-dm_coord_map", "Coordinate mapping for built-in mesh", "", DMPlexCoordMaps, (PetscEnum)map, (PetscEnum *)&map, &flg));
5130     if (flg) {
5131       switch (map) {
5132       case DM_COORD_MAP_NONE:
5133         mapFunc = coordMap_identity;
5134         break;
5135       case DM_COORD_MAP_SHEAR:
5136         mapFunc = coordMap_shear;
5137         if (!Np) {
5138           Np        = cdim + 1;
5139           params[0] = 0;
5140           for (PetscInt d = 1; d <= cdim; ++d) params[d] = 1.0;
5141         }
5142         PetscCheck(Np == cdim + 1, comm, PETSC_ERR_ARG_WRONG, "The shear coordinate map must have cdim + 1 = %" PetscInt_FMT " parameters, not %" PetscInt_FMT, cdim + 1, Np);
5143         break;
5144       case DM_COORD_MAP_FLARE:
5145         mapFunc = coordMap_flare;
5146         if (!Np) {
5147           Np        = cdim + 1;
5148           params[0] = 0;
5149           for (PetscInt d = 1; d <= cdim; ++d) params[d] = 1.0;
5150         }
5151         PetscCheck(Np == cdim + 1, comm, PETSC_ERR_ARG_WRONG, "The flare coordinate map must have cdim + 1 = %" PetscInt_FMT " parameters, not %" PetscInt_FMT, cdim + 1, Np);
5152         break;
5153       case DM_COORD_MAP_ANNULUS:
5154         mapFunc = coordMap_annulus;
5155         if (!Np) {
5156           Np        = 2;
5157           params[0] = 1.;
5158           params[1] = 2.;
5159         }
5160         PetscCheck(Np == 2, comm, PETSC_ERR_ARG_WRONG, "The annulus coordinate map must have 2 parameters, not %" PetscInt_FMT, Np);
5161         break;
5162       case DM_COORD_MAP_SHELL:
5163         mapFunc = coordMap_shell;
5164         if (!Np) {
5165           Np        = 2;
5166           params[0] = 1.;
5167           params[1] = 2.;
5168         }
5169         PetscCheck(Np == 2, comm, PETSC_ERR_ARG_WRONG, "The spherical shell coordinate map must have 2 parameters, not %" PetscInt_FMT, Np);
5170         break;
5171       default:
5172         mapFunc = coordMap_identity;
5173       }
5174     }
5175     if (Np) {
5176       DM      cdm;
5177       PetscDS cds;
5178 
5179       PetscCall(DMGetCoordinateDM(dm, &cdm));
5180       PetscCall(DMGetDS(cdm, &cds));
5181       PetscCall(PetscDSSetConstants(cds, Np, params));
5182     }
5183     PetscCall(DMPlexRemapGeometry(dm, 0.0, mapFunc));
5184   }
5185   /* Handle ghost cells */
5186   PetscCall(PetscOptionsBool("-dm_plex_create_fv_ghost_cells", "Flag to create finite volume ghost cells on the boundary", "DMCreate", ghostCells, &ghostCells, NULL));
5187   if (ghostCells) {
5188     DM   gdm;
5189     char lname[PETSC_MAX_PATH_LEN];
5190 
5191     lname[0] = '\0';
5192     PetscCall(PetscOptionsString("-dm_plex_fv_ghost_cells_label", "Label name for ghost cells boundary", "DMCreate", lname, lname, sizeof(lname), &flg));
5193     PetscCall(DMPlexConstructGhostCells(dm, flg ? lname : NULL, NULL, &gdm));
5194     PetscCall(DMPlexReplace_Internal(dm, &gdm));
5195   }
5196   /* Handle 1D order */
5197   if (reorder != DM_REORDER_DEFAULT_FALSE && dim == 1) {
5198     DM           cdm, rdm;
5199     PetscDS      cds;
5200     PetscObject  obj;
5201     PetscClassId id = PETSC_OBJECT_CLASSID;
5202     IS           perm;
5203     PetscInt     Nf;
5204     PetscBool    distributed;
5205 
5206     PetscCall(DMPlexIsDistributed(dm, &distributed));
5207     PetscCall(DMGetCoordinateDM(dm, &cdm));
5208     PetscCall(DMGetDS(cdm, &cds));
5209     PetscCall(PetscDSGetNumFields(cds, &Nf));
5210     if (Nf) {
5211       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
5212       PetscCall(PetscObjectGetClassId(obj, &id));
5213     }
5214     if (!distributed && id != PETSCFE_CLASSID) {
5215       PetscCall(DMPlexGetOrdering1D(dm, &perm));
5216       PetscCall(DMPlexPermute(dm, perm, &rdm));
5217       PetscCall(DMPlexReplace_Internal(dm, &rdm));
5218       PetscCall(ISDestroy(&perm));
5219     }
5220   }
5221 /* Handle */
5222 non_refine:
5223   PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5224   char    *phases[16];
5225   PetscInt Nphases = 16;
5226   PetscCall(PetscOptionsStringArray("-dm_plex_option_phases", "Option phase prefixes", "DMSetFromOptions", phases, &Nphases, &flg));
5227   PetscOptionsHeadEnd();
5228 
5229   // Phases
5230   if (flg) {
5231     const char *oldPrefix;
5232 
5233     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &oldPrefix));
5234     for (PetscInt ph = 0; ph < Nphases; ++ph) {
5235       PetscCall(PetscObjectAppendOptionsPrefix((PetscObject)dm, phases[ph]));
5236       PetscCall(PetscInfo(dm, "Options phase %s for DM %s\n", phases[ph], dm->hdr.name));
5237       PetscCall(DMSetFromOptions(dm));
5238       PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, oldPrefix));
5239       PetscCall(PetscFree(phases[ph]));
5240     }
5241   }
5242   PetscFunctionReturn(PETSC_SUCCESS);
5243 }
5244 
5245 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm, Vec *vec)
5246 {
5247   PetscFunctionBegin;
5248   PetscCall(DMCreateGlobalVector_Section_Private(dm, vec));
5249   /* PetscCall(VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM)); */
5250   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex));
5251   PetscCall(VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void))VecView_Plex_Native));
5252   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex));
5253   PetscCall(VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void))VecLoad_Plex_Native));
5254   PetscFunctionReturn(PETSC_SUCCESS);
5255 }
5256 
5257 static PetscErrorCode DMCreateLocalVector_Plex(DM dm, Vec *vec)
5258 {
5259   PetscFunctionBegin;
5260   PetscCall(DMCreateLocalVector_Section_Private(dm, vec));
5261   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex_Local));
5262   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex_Local));
5263   PetscFunctionReturn(PETSC_SUCCESS);
5264 }
5265 
5266 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
5267 {
5268   PetscInt depth, d;
5269 
5270   PetscFunctionBegin;
5271   PetscCall(DMPlexGetDepth(dm, &depth));
5272   if (depth == 1) {
5273     PetscCall(DMGetDimension(dm, &d));
5274     if (dim == 0) PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
5275     else if (dim == d) PetscCall(DMPlexGetDepthStratum(dm, 1, pStart, pEnd));
5276     else {
5277       *pStart = 0;
5278       *pEnd   = 0;
5279     }
5280   } else {
5281     PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
5282   }
5283   PetscFunctionReturn(PETSC_SUCCESS);
5284 }
5285 
5286 static PetscErrorCode DMGetNeighbors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
5287 {
5288   PetscSF            sf;
5289   PetscMPIInt        niranks, njranks;
5290   PetscInt           n;
5291   const PetscMPIInt *iranks, *jranks;
5292   DM_Plex           *data = (DM_Plex *)dm->data;
5293 
5294   PetscFunctionBegin;
5295   PetscCall(DMGetPointSF(dm, &sf));
5296   if (!data->neighbors) {
5297     PetscCall(PetscSFSetUp(sf));
5298     PetscCall(PetscSFGetRootRanks(sf, &njranks, &jranks, NULL, NULL, NULL));
5299     PetscCall(PetscSFGetLeafRanks(sf, &niranks, &iranks, NULL, NULL));
5300     PetscCall(PetscMalloc1(njranks + niranks + 1, &data->neighbors));
5301     PetscCall(PetscArraycpy(data->neighbors + 1, jranks, njranks));
5302     PetscCall(PetscArraycpy(data->neighbors + njranks + 1, iranks, niranks));
5303     n = njranks + niranks;
5304     PetscCall(PetscSortRemoveDupsMPIInt(&n, data->neighbors + 1));
5305     /* The following cast should never fail: can't have more neighbors than PETSC_MPI_INT_MAX */
5306     PetscCall(PetscMPIIntCast(n, data->neighbors));
5307   }
5308   if (nranks) *nranks = data->neighbors[0];
5309   if (ranks) {
5310     if (data->neighbors[0]) *ranks = data->neighbors + 1;
5311     else *ranks = NULL;
5312   }
5313   PetscFunctionReturn(PETSC_SUCCESS);
5314 }
5315 
5316 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM, DM, Mat, Vec, Vec);
5317 
5318 static PetscErrorCode DMInitialize_Plex(DM dm)
5319 {
5320   PetscFunctionBegin;
5321   dm->ops->view                      = DMView_Plex;
5322   dm->ops->load                      = DMLoad_Plex;
5323   dm->ops->setfromoptions            = DMSetFromOptions_Plex;
5324   dm->ops->clone                     = DMClone_Plex;
5325   dm->ops->setup                     = DMSetUp_Plex;
5326   dm->ops->createlocalsection        = DMCreateLocalSection_Plex;
5327   dm->ops->createsectionpermutation  = DMCreateSectionPermutation_Plex;
5328   dm->ops->createdefaultconstraints  = DMCreateDefaultConstraints_Plex;
5329   dm->ops->createglobalvector        = DMCreateGlobalVector_Plex;
5330   dm->ops->createlocalvector         = DMCreateLocalVector_Plex;
5331   dm->ops->getlocaltoglobalmapping   = NULL;
5332   dm->ops->createfieldis             = NULL;
5333   dm->ops->createcoordinatedm        = DMCreateCoordinateDM_Plex;
5334   dm->ops->createcoordinatefield     = DMCreateCoordinateField_Plex;
5335   dm->ops->getcoloring               = NULL;
5336   dm->ops->creatematrix              = DMCreateMatrix_Plex;
5337   dm->ops->createinterpolation       = DMCreateInterpolation_Plex;
5338   dm->ops->createmassmatrix          = DMCreateMassMatrix_Plex;
5339   dm->ops->createmassmatrixlumped    = DMCreateMassMatrixLumped_Plex;
5340   dm->ops->createinjection           = DMCreateInjection_Plex;
5341   dm->ops->refine                    = DMRefine_Plex;
5342   dm->ops->coarsen                   = DMCoarsen_Plex;
5343   dm->ops->refinehierarchy           = DMRefineHierarchy_Plex;
5344   dm->ops->coarsenhierarchy          = DMCoarsenHierarchy_Plex;
5345   dm->ops->extrude                   = DMExtrude_Plex;
5346   dm->ops->globaltolocalbegin        = NULL;
5347   dm->ops->globaltolocalend          = NULL;
5348   dm->ops->localtoglobalbegin        = NULL;
5349   dm->ops->localtoglobalend          = NULL;
5350   dm->ops->destroy                   = DMDestroy_Plex;
5351   dm->ops->createsubdm               = DMCreateSubDM_Plex;
5352   dm->ops->createsuperdm             = DMCreateSuperDM_Plex;
5353   dm->ops->getdimpoints              = DMGetDimPoints_Plex;
5354   dm->ops->locatepoints              = DMLocatePoints_Plex;
5355   dm->ops->projectfunctionlocal      = DMProjectFunctionLocal_Plex;
5356   dm->ops->projectfunctionlabellocal = DMProjectFunctionLabelLocal_Plex;
5357   dm->ops->projectfieldlocal         = DMProjectFieldLocal_Plex;
5358   dm->ops->projectfieldlabellocal    = DMProjectFieldLabelLocal_Plex;
5359   dm->ops->projectbdfieldlabellocal  = DMProjectBdFieldLabelLocal_Plex;
5360   dm->ops->computel2diff             = DMComputeL2Diff_Plex;
5361   dm->ops->computel2gradientdiff     = DMComputeL2GradientDiff_Plex;
5362   dm->ops->computel2fielddiff        = DMComputeL2FieldDiff_Plex;
5363   dm->ops->getneighbors              = DMGetNeighbors_Plex;
5364   dm->ops->getlocalboundingbox       = DMGetLocalBoundingBox_Coordinates;
5365   dm->ops->createdomaindecomposition = DMCreateDomainDecomposition_Plex;
5366   dm->ops->createddscatters          = DMCreateDomainDecompositionScatters_Plex;
5367   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertBoundaryValues_C", DMPlexInsertBoundaryValues_Plex));
5368   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertTimeDerivativeBoundaryValues_C", DMPlexInsertTimeDerivativeBoundaryValues_Plex));
5369   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMSetUpGLVisViewer_C", DMSetUpGLVisViewer_Plex));
5370   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMCreateNeumannOverlap_C", DMCreateNeumannOverlap_Plex));
5371   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeGetDefault_C", DMPlexDistributeGetDefault_Plex));
5372   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeSetDefault_C", DMPlexDistributeSetDefault_Plex));
5373   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderGetDefault_C", DMPlexReorderGetDefault_Plex));
5374   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderSetDefault_C", DMPlexReorderSetDefault_Plex));
5375   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionGetDefault_C", DMReorderSectionGetDefault_Plex));
5376   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionSetDefault_C", DMReorderSectionSetDefault_Plex));
5377   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionGetType_C", DMReorderSectionGetType_Plex));
5378   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionSetType_C", DMReorderSectionSetType_Plex));
5379   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMInterpolateSolution_C", DMInterpolateSolution_Plex));
5380   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
5381   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetOverlap_C", DMPlexSetOverlap_Plex));
5382   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetUseCeed_C", DMPlexGetUseCeed_Plex));
5383   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetUseCeed_C", DMPlexSetUseCeed_Plex));
5384   PetscFunctionReturn(PETSC_SUCCESS);
5385 }
5386 
5387 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
5388 {
5389   DM_Plex       *mesh = (DM_Plex *)dm->data;
5390   const PetscSF *face_sfs;
5391   PetscInt       num_face_sfs;
5392 
5393   PetscFunctionBegin;
5394   mesh->refct++;
5395   (*newdm)->data = mesh;
5396   PetscCall(DMPlexGetIsoperiodicFaceSF(dm, &num_face_sfs, &face_sfs));
5397   PetscCall(DMPlexSetIsoperiodicFaceSF(*newdm, num_face_sfs, (PetscSF *)face_sfs));
5398   PetscCall(PetscObjectChangeTypeName((PetscObject)*newdm, DMPLEX));
5399   PetscCall(DMInitialize_Plex(*newdm));
5400   PetscFunctionReturn(PETSC_SUCCESS);
5401 }
5402 
5403 /*MC
5404   DMPLEX = "plex" - A `DM` object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
5405                     In the local representation, `Vec`s contain all unknowns in the interior and shared boundary. This is
5406                     specified by a PetscSection object. Ownership in the global representation is determined by
5407                     ownership of the underlying `DMPLEX` points. This is specified by another `PetscSection` object.
5408 
5409   Options Database Keys:
5410 + -dm_refine_pre                     - Refine mesh before distribution
5411 + -dm_refine_uniform_pre             - Choose uniform or generator-based refinement
5412 + -dm_refine_volume_limit_pre        - Cell volume limit after pre-refinement using generator
5413 . -dm_distribute                     - Distribute mesh across processes
5414 . -dm_distribute_overlap             - Number of cells to overlap for distribution
5415 . -dm_refine                         - Refine mesh after distribution
5416 . -dm_localize <bool>                - Whether to localize coordinates for periodic meshes
5417 . -dm_sparse_localize <bool>         - Whether to only localize cells on the periodic boundary
5418 . -dm_plex_hash_location             - Use grid hashing for point location
5419 . -dm_plex_hash_box_faces <n,m,p>    - The number of divisions in each direction of the grid hash
5420 . -dm_plex_partition_balance         - Attempt to evenly divide points on partition boundary between processes
5421 . -dm_plex_remesh_bd                 - Allow changes to the boundary on remeshing
5422 . -dm_plex_max_projection_height     - Maximum mesh point height used to project locally
5423 . -dm_plex_regular_refinement        - Use special nested projection algorithm for regular refinement
5424 . -dm_plex_reorder_section           - Use specialized blocking if available
5425 . -dm_plex_check_all                 - Perform all checks below
5426 . -dm_plex_check_symmetry            - Check that the adjacency information in the mesh is symmetric
5427 . -dm_plex_check_skeleton <celltype> - Check that each cell has the correct number of vertices
5428 . -dm_plex_check_faces <celltype>    - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
5429 . -dm_plex_check_geometry            - Check that cells have positive volume
5430 . -dm_view :mesh.tex:ascii_latex     - View the mesh in LaTeX/TikZ
5431 . -dm_plex_view_scale <num>          - Scale the TikZ
5432 . -dm_plex_print_fem <num>           - View FEM assembly information, such as element vectors and matrices
5433 - -dm_plex_print_fvm <num>           - View FVM assembly information, such as flux updates
5434 
5435   Level: intermediate
5436 
5437 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`, `PetscSection`
5438 M*/
5439 
5440 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
5441 {
5442   DM_Plex *mesh;
5443   PetscInt unit;
5444 
5445   PetscFunctionBegin;
5446   PetscCall(PetscCitationsRegister(PlexCitation, &Plexcite));
5447   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5448   PetscCall(PetscNew(&mesh));
5449   dm->reorderSection = DM_REORDER_DEFAULT_NOTSET;
5450   dm->data           = mesh;
5451 
5452   mesh->refct = 1;
5453   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection));
5454   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection));
5455   mesh->refinementUniform      = PETSC_TRUE;
5456   mesh->refinementLimit        = -1.0;
5457   mesh->distDefault            = PETSC_TRUE;
5458   mesh->reorderDefault         = DM_REORDER_DEFAULT_NOTSET;
5459   mesh->distributionName       = NULL;
5460   mesh->interpolated           = DMPLEX_INTERPOLATED_INVALID;
5461   mesh->interpolatedCollective = DMPLEX_INTERPOLATED_INVALID;
5462 
5463   PetscCall(PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner));
5464   mesh->remeshBd = PETSC_FALSE;
5465 
5466   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
5467 
5468   mesh->depthState    = -1;
5469   mesh->celltypeState = -1;
5470   mesh->printTol      = 1.0e-10;
5471   mesh->nonempty_comm = MPI_COMM_SELF;
5472 
5473   PetscCall(DMInitialize_Plex(dm));
5474   PetscFunctionReturn(PETSC_SUCCESS);
5475 }
5476 
5477 /*@
5478   DMPlexCreate - Creates a `DMPLEX` object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
5479 
5480   Collective
5481 
5482   Input Parameter:
5483 . comm - The communicator for the `DMPLEX` object
5484 
5485   Output Parameter:
5486 . mesh - The `DMPLEX` object
5487 
5488   Level: beginner
5489 
5490 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMType`, `DMCreate()`, `DMSetType()`
5491 @*/
5492 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
5493 {
5494   PetscFunctionBegin;
5495   PetscAssertPointer(mesh, 2);
5496   PetscCall(DMCreate(comm, mesh));
5497   PetscCall(DMSetType(*mesh, DMPLEX));
5498   PetscFunctionReturn(PETSC_SUCCESS);
5499 }
5500 
5501 /*@C
5502   DMPlexBuildFromCellListParallel - Build distributed `DMPLEX` topology from a list of vertices for each cell (common mesh generator output) where all cells have the same celltype
5503 
5504   Collective; No Fortran Support
5505 
5506   Input Parameters:
5507 + dm          - The `DM`
5508 . numCells    - The number of cells owned by this process
5509 . numVertices - The number of vertices to be owned by this process, or `PETSC_DECIDE`
5510 . NVertices   - The global number of vertices, or `PETSC_DETERMINE`
5511 . numCorners  - The number of vertices for each cell
5512 - cells       - An array of numCells*numCorners numbers, the global vertex numbers for each cell
5513 
5514   Output Parameters:
5515 + vertexSF         - (Optional) `PetscSF` describing complete vertex ownership
5516 - verticesAdjSaved - (Optional) vertex adjacency array
5517 
5518   Level: advanced
5519 
5520   Notes:
5521   Two triangles sharing a face
5522 .vb
5523 
5524         2
5525       / | \
5526      /  |  \
5527     /   |   \
5528    0  0 | 1  3
5529     \   |   /
5530      \  |  /
5531       \ | /
5532         1
5533 .ve
5534   would have input
5535 .vb
5536   numCells = 2, numVertices = 4
5537   cells = [0 1 2  1 3 2]
5538 .ve
5539   which would result in the `DMPLEX`
5540 .vb
5541 
5542         4
5543       / | \
5544      /  |  \
5545     /   |   \
5546    2  0 | 1  5
5547     \   |   /
5548      \  |  /
5549       \ | /
5550         3
5551 .ve
5552 
5553   Vertices are implicitly numbered consecutively 0,...,NVertices.
5554   Each rank owns a chunk of numVertices consecutive vertices.
5555   If numVertices is `PETSC_DECIDE`, PETSc will distribute them as evenly as possible using PetscLayout.
5556   If NVertices is `PETSC_DETERMINE` and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
5557   If only NVertices is `PETSC_DETERMINE`, it is computed as the sum of numVertices over all ranks.
5558 
5559   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
5560 
5561 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildCoordinatesFromCellListParallel()`,
5562           `PetscSF`
5563 @*/
5564 PetscErrorCode DMPlexBuildFromCellListParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, const PetscInt cells[], PetscSF *vertexSF, PetscInt **verticesAdjSaved)
5565 {
5566   PetscSF     sfPoint;
5567   PetscLayout layout;
5568   PetscInt    numVerticesAdj, *verticesAdj, *cones, c, p;
5569 
5570   PetscFunctionBegin;
5571   PetscValidLogicalCollectiveInt(dm, NVertices, 4);
5572   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5573   /* Get/check global number of vertices */
5574   {
5575     PetscInt       NVerticesInCells, i;
5576     const PetscInt len = numCells * numCorners;
5577 
5578     /* NVerticesInCells = max(cells) + 1 */
5579     NVerticesInCells = PETSC_INT_MIN;
5580     for (i = 0; i < len; i++)
5581       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
5582     ++NVerticesInCells;
5583     PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
5584 
5585     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
5586     else
5587       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
5588   }
5589   /* Count locally unique vertices */
5590   {
5591     PetscHSetI vhash;
5592     PetscInt   off = 0;
5593 
5594     PetscCall(PetscHSetICreate(&vhash));
5595     for (c = 0; c < numCells; ++c) {
5596       for (p = 0; p < numCorners; ++p) PetscCall(PetscHSetIAdd(vhash, cells[c * numCorners + p]));
5597     }
5598     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
5599     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
5600     else verticesAdj = *verticesAdjSaved;
5601     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
5602     PetscCall(PetscHSetIDestroy(&vhash));
5603     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
5604   }
5605   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
5606   /* Create cones */
5607   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
5608   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
5609   PetscCall(DMSetUp(dm));
5610   PetscCall(DMPlexGetCones(dm, &cones));
5611   for (c = 0; c < numCells; ++c) {
5612     for (p = 0; p < numCorners; ++p) {
5613       const PetscInt gv = cells[c * numCorners + p];
5614       PetscInt       lv;
5615 
5616       /* Positions within verticesAdj form 0-based local vertex numbering;
5617          we need to shift it by numCells to get correct DAG points (cells go first) */
5618       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
5619       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
5620       cones[c * numCorners + p] = lv + numCells;
5621     }
5622   }
5623   /* Build point sf */
5624   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
5625   PetscCall(PetscLayoutSetSize(layout, NVertices));
5626   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
5627   PetscCall(PetscLayoutSetBlockSize(layout, 1));
5628   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
5629   PetscCall(PetscLayoutDestroy(&layout));
5630   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
5631   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
5632   if (dm->sf) {
5633     const char *prefix;
5634 
5635     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
5636     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
5637   }
5638   PetscCall(DMSetPointSF(dm, sfPoint));
5639   PetscCall(PetscSFDestroy(&sfPoint));
5640   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)*vertexSF, "Vertex Ownership SF"));
5641   /* Fill in the rest of the topology structure */
5642   PetscCall(DMPlexSymmetrize(dm));
5643   PetscCall(DMPlexStratify(dm));
5644   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5645   PetscFunctionReturn(PETSC_SUCCESS);
5646 }
5647 
5648 /*@C
5649   DMPlexBuildFromCellSectionParallel - Build distributed `DMPLEX` topology from a list of vertices for each cell (common mesh generator output) allowing multiple celltypes
5650 
5651   Collective; No Fortran Support
5652 
5653   Input Parameters:
5654 + dm          - The `DM`
5655 . numCells    - The number of cells owned by this process
5656 . numVertices - The number of vertices to be owned by this process, or `PETSC_DECIDE`
5657 . NVertices   - The global number of vertices, or `PETSC_DETERMINE`
5658 . cellSection - The `PetscSection` giving the number of vertices for each cell (layout of cells)
5659 - cells       - An array of the global vertex numbers for each cell
5660 
5661   Output Parameters:
5662 + vertexSF         - (Optional) `PetscSF` describing complete vertex ownership
5663 - verticesAdjSaved - (Optional) vertex adjacency array
5664 
5665   Level: advanced
5666 
5667   Notes:
5668   A triangle and quadrilateral sharing a face
5669 .vb
5670         2----------3
5671       / |          |
5672      /  |          |
5673     /   |          |
5674    0  0 |     1    |
5675     \   |          |
5676      \  |          |
5677       \ |          |
5678         1----------4
5679 .ve
5680   would have input
5681 .vb
5682   numCells = 2, numVertices = 5
5683   cells = [0 1 2  1 4 3 2]
5684 .ve
5685   which would result in the `DMPLEX`
5686 .vb
5687         4----------5
5688       / |          |
5689      /  |          |
5690     /   |          |
5691    2  0 |     1    |
5692     \   |          |
5693      \  |          |
5694       \ |          |
5695         3----------6
5696 .ve
5697 
5698   Vertices are implicitly numbered consecutively 0,...,NVertices.
5699   Each rank owns a chunk of numVertices consecutive vertices.
5700   If numVertices is `PETSC_DECIDE`, PETSc will distribute them as evenly as possible using PetscLayout.
5701   If NVertices is `PETSC_DETERMINE` and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
5702   If only NVertices is `PETSC_DETERMINE`, it is computed as the sum of numVertices over all ranks.
5703 
5704   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
5705 
5706 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellListParallel()`, `DMPlexCreateFromCellSectionParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`,
5707           `PetscSF`
5708 @*/
5709 PetscErrorCode DMPlexBuildFromCellSectionParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscSection cellSection, const PetscInt cells[], PetscSF *vertexSF, PetscInt **verticesAdjSaved)
5710 {
5711   PetscSF     sfPoint;
5712   PetscLayout layout;
5713   PetscInt    numVerticesAdj, *verticesAdj, *cones, cStart, cEnd, len;
5714 
5715   PetscFunctionBegin;
5716   PetscValidLogicalCollectiveInt(dm, NVertices, 4);
5717   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5718   PetscCall(PetscSectionGetChart(cellSection, &cStart, &cEnd));
5719   PetscCall(PetscSectionGetStorageSize(cellSection, &len));
5720   PetscCheck(cStart == 0 && cEnd == numCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Section chart [%" PetscInt_FMT ", %" PetscInt_FMT ") should be [0, %" PetscInt_FMT ")", cStart, cEnd, numCells);
5721   /* Get/check global number of vertices */
5722   {
5723     PetscInt NVerticesInCells;
5724 
5725     /* NVerticesInCells = max(cells) + 1 */
5726     NVerticesInCells = PETSC_MIN_INT;
5727     for (PetscInt i = 0; i < len; i++)
5728       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
5729     ++NVerticesInCells;
5730     PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
5731 
5732     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
5733     else
5734       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
5735   }
5736   /* Count locally unique vertices */
5737   {
5738     PetscHSetI vhash;
5739     PetscInt   off = 0;
5740 
5741     PetscCall(PetscHSetICreate(&vhash));
5742     for (PetscInt i = 0; i < len; i++) PetscCall(PetscHSetIAdd(vhash, cells[i]));
5743     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
5744     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
5745     else verticesAdj = *verticesAdjSaved;
5746     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
5747     PetscCall(PetscHSetIDestroy(&vhash));
5748     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
5749   }
5750   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
5751   /* Create cones */
5752   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
5753   for (PetscInt c = 0; c < numCells; ++c) {
5754     PetscInt dof;
5755 
5756     PetscCall(PetscSectionGetDof(cellSection, c, &dof));
5757     PetscCall(DMPlexSetConeSize(dm, c, dof));
5758   }
5759   PetscCall(DMSetUp(dm));
5760   PetscCall(DMPlexGetCones(dm, &cones));
5761   for (PetscInt c = 0; c < numCells; ++c) {
5762     PetscInt dof, off;
5763 
5764     PetscCall(PetscSectionGetDof(cellSection, c, &dof));
5765     PetscCall(PetscSectionGetOffset(cellSection, c, &off));
5766     for (PetscInt p = off; p < off + dof; ++p) {
5767       const PetscInt gv = cells[p];
5768       PetscInt       lv;
5769 
5770       /* Positions within verticesAdj form 0-based local vertex numbering;
5771          we need to shift it by numCells to get correct DAG points (cells go first) */
5772       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
5773       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
5774       cones[p] = lv + numCells;
5775     }
5776   }
5777   /* Build point sf */
5778   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
5779   PetscCall(PetscLayoutSetSize(layout, NVertices));
5780   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
5781   PetscCall(PetscLayoutSetBlockSize(layout, 1));
5782   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
5783   PetscCall(PetscLayoutDestroy(&layout));
5784   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
5785   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
5786   if (dm->sf) {
5787     const char *prefix;
5788 
5789     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
5790     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
5791   }
5792   PetscCall(DMSetPointSF(dm, sfPoint));
5793   PetscCall(PetscSFDestroy(&sfPoint));
5794   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)*vertexSF, "Vertex Ownership SF"));
5795   /* Fill in the rest of the topology structure */
5796   PetscCall(DMPlexSymmetrize(dm));
5797   PetscCall(DMPlexStratify(dm));
5798   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5799   PetscFunctionReturn(PETSC_SUCCESS);
5800 }
5801 
5802 /*@
5803   DMPlexBuildCoordinatesFromCellListParallel - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
5804 
5805   Collective; No Fortran Support
5806 
5807   Input Parameters:
5808 + dm           - The `DM`
5809 . spaceDim     - The spatial dimension used for coordinates
5810 . sfVert       - `PetscSF` describing complete vertex ownership
5811 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
5812 
5813   Level: advanced
5814 
5815 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellListParallel()`
5816 @*/
5817 PetscErrorCode DMPlexBuildCoordinatesFromCellListParallel(DM dm, PetscInt spaceDim, PetscSF sfVert, const PetscReal vertexCoords[])
5818 {
5819   PetscSection coordSection;
5820   Vec          coordinates;
5821   PetscScalar *coords;
5822   PetscInt     numVertices, numVerticesAdj, coordSize, v, vStart, vEnd;
5823   PetscMPIInt  spaceDimi;
5824 
5825   PetscFunctionBegin;
5826   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
5827   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
5828   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
5829   PetscCall(DMSetCoordinateDim(dm, spaceDim));
5830   PetscCall(PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL));
5831   PetscCheck(vEnd - vStart == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Supplied sfVert has wrong number of leaves = %" PetscInt_FMT " != %" PetscInt_FMT " = vEnd - vStart", numVerticesAdj, vEnd - vStart);
5832   PetscCall(DMGetCoordinateSection(dm, &coordSection));
5833   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5834   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
5835   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
5836   for (v = vStart; v < vEnd; ++v) {
5837     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
5838     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
5839   }
5840   PetscCall(PetscSectionSetUp(coordSection));
5841   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
5842   PetscCall(VecCreate(PetscObjectComm((PetscObject)dm), &coordinates));
5843   PetscCall(VecSetBlockSize(coordinates, spaceDim));
5844   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5845   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
5846   PetscCall(VecSetType(coordinates, VECSTANDARD));
5847   PetscCall(VecGetArray(coordinates, &coords));
5848   {
5849     MPI_Datatype coordtype;
5850 
5851     /* Need a temp buffer for coords if we have complex/single */
5852     PetscCall(PetscMPIIntCast(spaceDim, &spaceDimi));
5853     PetscCallMPI(MPI_Type_contiguous(spaceDimi, MPIU_SCALAR, &coordtype));
5854     PetscCallMPI(MPI_Type_commit(&coordtype));
5855 #if defined(PETSC_USE_COMPLEX)
5856     {
5857       PetscScalar *svertexCoords;
5858       PetscInt     i;
5859       PetscCall(PetscMalloc1(numVertices * spaceDim, &svertexCoords));
5860       for (i = 0; i < numVertices * spaceDim; i++) svertexCoords[i] = vertexCoords[i];
5861       PetscCall(PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
5862       PetscCall(PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
5863       PetscCall(PetscFree(svertexCoords));
5864     }
5865 #else
5866     PetscCall(PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
5867     PetscCall(PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
5868 #endif
5869     PetscCallMPI(MPI_Type_free(&coordtype));
5870   }
5871   PetscCall(VecRestoreArray(coordinates, &coords));
5872   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
5873   PetscCall(VecDestroy(&coordinates));
5874   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
5875   PetscFunctionReturn(PETSC_SUCCESS);
5876 }
5877 
5878 /*@
5879   DMPlexCreateFromCellListParallelPetsc - Create distributed `DMPLEX` from a list of vertices for each cell (common mesh generator output) where all cells have the same celltype
5880 
5881   Collective
5882 
5883   Input Parameters:
5884 + comm         - The communicator
5885 . dim          - The topological dimension of the mesh
5886 . numCells     - The number of cells owned by this process
5887 . numVertices  - The number of vertices owned by this process, or `PETSC_DECIDE`
5888 . NVertices    - The global number of vertices, or `PETSC_DECIDE`
5889 . numCorners   - The number of vertices for each cell
5890 . interpolate  - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
5891 . cells        - An array of numCells*numCorners numbers, the global vertex numbers for each cell
5892 . spaceDim     - The spatial dimension used for coordinates
5893 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
5894 
5895   Output Parameters:
5896 + dm          - The `DM`
5897 . vertexSF    - (Optional) `PetscSF` describing complete vertex ownership
5898 - verticesAdj - (Optional) vertex adjacency array
5899 
5900   Level: intermediate
5901 
5902   Notes:
5903   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`,
5904   `DMPlexBuildFromCellListParallel()`, `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellListParallel()`
5905 
5906   See `DMPlexBuildFromCellListParallel()` for an example and details about the topology-related parameters.
5907 
5908   See `DMPlexBuildCoordinatesFromCellListParallel()` for details about the geometry-related parameters.
5909 
5910 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
5911 @*/
5912 PetscErrorCode DMPlexCreateFromCellListParallelPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, PetscInt **verticesAdj, DM *dm)
5913 {
5914   PetscSF sfVert;
5915 
5916   PetscFunctionBegin;
5917   PetscCall(DMCreate(comm, dm));
5918   PetscCall(DMSetType(*dm, DMPLEX));
5919   PetscValidLogicalCollectiveInt(*dm, dim, 2);
5920   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
5921   PetscCall(DMSetDimension(*dm, dim));
5922   PetscCall(DMPlexBuildFromCellListParallel(*dm, numCells, numVertices, NVertices, numCorners, cells, &sfVert, verticesAdj));
5923   if (interpolate) {
5924     DM idm;
5925 
5926     PetscCall(DMPlexInterpolate(*dm, &idm));
5927     PetscCall(DMDestroy(dm));
5928     *dm = idm;
5929   }
5930   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
5931   if (vertexSF) *vertexSF = sfVert;
5932   else PetscCall(PetscSFDestroy(&sfVert));
5933   PetscFunctionReturn(PETSC_SUCCESS);
5934 }
5935 
5936 /*@
5937   DMPlexCreateFromCellSectionParallel - Create distributed `DMPLEX` from a list of vertices for each cell (common mesh generator output) and supports multiple celltypes
5938 
5939   Collective
5940 
5941   Input Parameters:
5942 + comm         - The communicator
5943 . dim          - The topological dimension of the mesh
5944 . numCells     - The number of cells owned by this process
5945 . numVertices  - The number of vertices owned by this process, or `PETSC_DECIDE`
5946 . NVertices    - The global number of vertices, or `PETSC_DECIDE`
5947 . cellSection  - The `PetscSection` giving the number of vertices for each cell (layout of cells)
5948 . interpolate  - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
5949 . cells        - An array of the global vertex numbers for each cell
5950 . spaceDim     - The spatial dimension used for coordinates
5951 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
5952 
5953   Output Parameters:
5954 + dm          - The `DM`
5955 . vertexSF    - (Optional) `PetscSF` describing complete vertex ownership
5956 - verticesAdj - (Optional) vertex adjacency array
5957 
5958   Level: intermediate
5959 
5960   Notes:
5961   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`,
5962   `DMPlexBuildFromCellSectionParallel()`, `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellListParallel()`
5963 
5964   See `DMPlexBuildFromCellSectionParallel()` for an example and details about the topology-related parameters.
5965 
5966   See `DMPlexBuildCoordinatesFromCellListParallel()` for details about the geometry-related parameters.
5967 
5968 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
5969 @*/
5970 PetscErrorCode DMPlexCreateFromCellSectionParallel(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscSection cellSection, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, PetscInt **verticesAdj, DM *dm)
5971 {
5972   PetscSF sfVert;
5973 
5974   PetscFunctionBegin;
5975   PetscCall(DMCreate(comm, dm));
5976   PetscCall(DMSetType(*dm, DMPLEX));
5977   PetscValidLogicalCollectiveInt(*dm, dim, 2);
5978   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
5979   PetscCall(DMSetDimension(*dm, dim));
5980   PetscCall(DMPlexBuildFromCellSectionParallel(*dm, numCells, numVertices, NVertices, cellSection, cells, &sfVert, verticesAdj));
5981   if (interpolate) {
5982     DM idm;
5983 
5984     PetscCall(DMPlexInterpolate(*dm, &idm));
5985     PetscCall(DMDestroy(dm));
5986     *dm = idm;
5987   }
5988   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
5989   if (vertexSF) *vertexSF = sfVert;
5990   else PetscCall(PetscSFDestroy(&sfVert));
5991   PetscFunctionReturn(PETSC_SUCCESS);
5992 }
5993 
5994 /*@
5995   DMPlexBuildFromCellList - Build `DMPLEX` topology from a list of vertices for each cell (common mesh generator output)
5996 
5997   Collective; No Fortran Support
5998 
5999   Input Parameters:
6000 + dm          - The `DM`
6001 . numCells    - The number of cells owned by this process
6002 . numVertices - The number of vertices owned by this process, or `PETSC_DETERMINE`
6003 . numCorners  - The number of vertices for each cell
6004 - cells       - An array of `numCells` x `numCorners` numbers, the global vertex numbers for each cell
6005 
6006   Level: advanced
6007 
6008   Notes:
6009   Two triangles sharing a face
6010 .vb
6011 
6012         2
6013       / | \
6014      /  |  \
6015     /   |   \
6016    0  0 | 1  3
6017     \   |   /
6018      \  |  /
6019       \ | /
6020         1
6021 .ve
6022   would have input
6023 .vb
6024   numCells = 2, numVertices = 4
6025   cells = [0 1 2  1 3 2]
6026 .ve
6027   which would result in the `DMPLEX`
6028 .vb
6029 
6030         4
6031       / | \
6032      /  |  \
6033     /   |   \
6034    2  0 | 1  5
6035     \   |   /
6036      \  |  /
6037       \ | /
6038         3
6039 .ve
6040 
6041   If numVertices is `PETSC_DETERMINE`, it is computed by PETSc as the maximum vertex index in cells + 1.
6042 
6043 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListPetsc()`
6044 @*/
6045 PetscErrorCode DMPlexBuildFromCellList(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const PetscInt cells[])
6046 {
6047   PetscInt *cones, c, p, dim;
6048 
6049   PetscFunctionBegin;
6050   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
6051   PetscCall(DMGetDimension(dm, &dim));
6052   /* Get/check global number of vertices */
6053   {
6054     PetscInt       NVerticesInCells, i;
6055     const PetscInt len = numCells * numCorners;
6056 
6057     /* NVerticesInCells = max(cells) + 1 */
6058     NVerticesInCells = PETSC_INT_MIN;
6059     for (i = 0; i < len; i++)
6060       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
6061     ++NVerticesInCells;
6062 
6063     if (numVertices == PETSC_DECIDE) numVertices = NVerticesInCells;
6064     else
6065       PetscCheck(numVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, numVertices, NVerticesInCells);
6066   }
6067   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
6068   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
6069   PetscCall(DMSetUp(dm));
6070   PetscCall(DMPlexGetCones(dm, &cones));
6071   for (c = 0; c < numCells; ++c) {
6072     for (p = 0; p < numCorners; ++p) cones[c * numCorners + p] = cells[c * numCorners + p] + numCells;
6073   }
6074   PetscCall(DMPlexSymmetrize(dm));
6075   PetscCall(DMPlexStratify(dm));
6076   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
6077   PetscFunctionReturn(PETSC_SUCCESS);
6078 }
6079 
6080 /*@
6081   DMPlexBuildCoordinatesFromCellList - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
6082 
6083   Collective
6084 
6085   Input Parameters:
6086 + dm           - The `DM`
6087 . spaceDim     - The spatial dimension used for coordinates
6088 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
6089 
6090   Level: advanced
6091 
6092 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellList()`
6093 @*/
6094 PetscErrorCode DMPlexBuildCoordinatesFromCellList(DM dm, PetscInt spaceDim, const PetscReal vertexCoords[])
6095 {
6096   PetscSection coordSection;
6097   Vec          coordinates;
6098   DM           cdm;
6099   PetscScalar *coords;
6100   PetscInt     v, vStart, vEnd, d;
6101 
6102   PetscFunctionBegin;
6103   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
6104   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
6105   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
6106   PetscCall(DMSetCoordinateDim(dm, spaceDim));
6107   PetscCall(DMGetCoordinateSection(dm, &coordSection));
6108   PetscCall(PetscSectionSetNumFields(coordSection, 1));
6109   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
6110   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
6111   for (v = vStart; v < vEnd; ++v) {
6112     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
6113     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
6114   }
6115   PetscCall(PetscSectionSetUp(coordSection));
6116 
6117   PetscCall(DMGetCoordinateDM(dm, &cdm));
6118   PetscCall(DMCreateLocalVector(cdm, &coordinates));
6119   PetscCall(VecSetBlockSize(coordinates, spaceDim));
6120   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
6121   PetscCall(VecGetArrayWrite(coordinates, &coords));
6122   for (v = 0; v < vEnd - vStart; ++v) {
6123     for (d = 0; d < spaceDim; ++d) coords[v * spaceDim + d] = vertexCoords[v * spaceDim + d];
6124   }
6125   PetscCall(VecRestoreArrayWrite(coordinates, &coords));
6126   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
6127   PetscCall(VecDestroy(&coordinates));
6128   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
6129   PetscFunctionReturn(PETSC_SUCCESS);
6130 }
6131 
6132 /*@
6133   DMPlexCreateFromCellListPetsc - Create `DMPLEX` from a list of vertices for each cell (common mesh generator output), but only process 0 takes in the input
6134 
6135   Collective
6136 
6137   Input Parameters:
6138 + comm         - The communicator
6139 . dim          - The topological dimension of the mesh
6140 . numCells     - The number of cells, only on process 0
6141 . numVertices  - The number of vertices owned by this process, or `PETSC_DECIDE`, only on process 0
6142 . numCorners   - The number of vertices for each cell, only on process 0
6143 . interpolate  - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
6144 . cells        - An array of numCells*numCorners numbers, the vertices for each cell, only on process 0
6145 . spaceDim     - The spatial dimension used for coordinates
6146 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex, only on process 0
6147 
6148   Output Parameter:
6149 . dm - The `DM`, which only has points on process 0
6150 
6151   Level: intermediate
6152 
6153   Notes:
6154   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`, `DMPlexBuildFromCellList()`,
6155   `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellList()`
6156 
6157   See `DMPlexBuildFromCellList()` for an example and details about the topology-related parameters.
6158   See `DMPlexBuildCoordinatesFromCellList()` for details about the geometry-related parameters.
6159   See `DMPlexCreateFromCellListParallelPetsc()` for parallel input
6160 
6161 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellList()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
6162 @*/
6163 PetscErrorCode DMPlexCreateFromCellListPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], DM *dm)
6164 {
6165   PetscMPIInt rank;
6166 
6167   PetscFunctionBegin;
6168   PetscCheck(dim, comm, PETSC_ERR_ARG_OUTOFRANGE, "This is not appropriate for 0-dimensional meshes. Consider either creating the DM using DMPlexCreateFromDAG(), by hand, or using DMSwarm.");
6169   PetscCallMPI(MPI_Comm_rank(comm, &rank));
6170   PetscCall(DMCreate(comm, dm));
6171   PetscCall(DMSetType(*dm, DMPLEX));
6172   PetscCall(DMSetDimension(*dm, dim));
6173   if (rank == 0) PetscCall(DMPlexBuildFromCellList(*dm, numCells, numVertices, numCorners, cells));
6174   else PetscCall(DMPlexBuildFromCellList(*dm, 0, 0, 0, NULL));
6175   if (interpolate) {
6176     DM idm;
6177 
6178     PetscCall(DMPlexInterpolate(*dm, &idm));
6179     PetscCall(DMDestroy(dm));
6180     *dm = idm;
6181   }
6182   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, vertexCoords));
6183   else PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, NULL));
6184   PetscFunctionReturn(PETSC_SUCCESS);
6185 }
6186 
6187 /*@
6188   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a `DM`
6189 
6190   Input Parameters:
6191 + dm               - The empty `DM` object, usually from `DMCreate()` and `DMSetDimension()`
6192 . depth            - The depth of the DAG
6193 . numPoints        - Array of size depth + 1 containing the number of points at each `depth`
6194 . coneSize         - The cone size of each point
6195 . cones            - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
6196 . coneOrientations - The orientation of each cone point
6197 - vertexCoords     - An array of `numPoints`[0]*spacedim numbers representing the coordinates of each vertex, with spacedim the value set via `DMSetCoordinateDim()`
6198 
6199   Output Parameter:
6200 . dm - The `DM`
6201 
6202   Level: advanced
6203 
6204   Note:
6205   Two triangles sharing a face would have input
6206 .vb
6207   depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
6208   cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
6209  vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
6210 .ve
6211   which would result in the DMPlex
6212 .vb
6213         4
6214       / | \
6215      /  |  \
6216     /   |   \
6217    2  0 | 1  5
6218     \   |   /
6219      \  |  /
6220       \ | /
6221         3
6222 .ve
6223   Notice that all points are numbered consecutively, unlike `DMPlexCreateFromCellListPetsc()`
6224 
6225 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
6226 @*/
6227 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
6228 {
6229   Vec          coordinates;
6230   PetscSection coordSection;
6231   PetscScalar *coords;
6232   PetscInt     coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
6233 
6234   PetscFunctionBegin;
6235   PetscCall(DMGetDimension(dm, &dim));
6236   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
6237   PetscCheck(dimEmbed >= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Embedding dimension %" PetscInt_FMT " cannot be less than intrinsic dimension %" PetscInt_FMT, dimEmbed, dim);
6238   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
6239   PetscCall(DMPlexSetChart(dm, pStart, pEnd));
6240   for (p = pStart; p < pEnd; ++p) {
6241     PetscCall(DMPlexSetConeSize(dm, p, coneSize[p - pStart]));
6242     if (firstVertex < 0 && !coneSize[p - pStart]) firstVertex = p - pStart;
6243   }
6244   PetscCheck(firstVertex >= 0 || !numPoints[0], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Expected %" PetscInt_FMT " vertices but could not find any", numPoints[0]);
6245   PetscCall(DMSetUp(dm)); /* Allocate space for cones */
6246   for (p = pStart, off = 0; p < pEnd; off += coneSize[p - pStart], ++p) {
6247     PetscCall(DMPlexSetCone(dm, p, &cones[off]));
6248     PetscCall(DMPlexSetConeOrientation(dm, p, &coneOrientations[off]));
6249   }
6250   PetscCall(DMPlexSymmetrize(dm));
6251   PetscCall(DMPlexStratify(dm));
6252   /* Build coordinates */
6253   PetscCall(DMGetCoordinateSection(dm, &coordSection));
6254   PetscCall(PetscSectionSetNumFields(coordSection, 1));
6255   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dimEmbed));
6256   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numPoints[0]));
6257   for (v = firstVertex; v < firstVertex + numPoints[0]; ++v) {
6258     PetscCall(PetscSectionSetDof(coordSection, v, dimEmbed));
6259     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed));
6260   }
6261   PetscCall(PetscSectionSetUp(coordSection));
6262   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
6263   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
6264   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
6265   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
6266   PetscCall(VecSetBlockSize(coordinates, dimEmbed));
6267   PetscCall(VecSetType(coordinates, VECSTANDARD));
6268   if (vertexCoords) {
6269     PetscCall(VecGetArray(coordinates, &coords));
6270     for (v = 0; v < numPoints[0]; ++v) {
6271       PetscInt off;
6272 
6273       PetscCall(PetscSectionGetOffset(coordSection, v + firstVertex, &off));
6274       for (d = 0; d < dimEmbed; ++d) coords[off + d] = vertexCoords[v * dimEmbed + d];
6275     }
6276   }
6277   PetscCall(VecRestoreArray(coordinates, &coords));
6278   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
6279   PetscCall(VecDestroy(&coordinates));
6280   PetscFunctionReturn(PETSC_SUCCESS);
6281 }
6282 
6283 /*
6284   DMPlexCreateCellVertexFromFile - Create a `DMPLEX` mesh from a simple cell-vertex file.
6285 
6286   Collective
6287 
6288 + comm        - The MPI communicator
6289 . filename    - Name of the .dat file
6290 - interpolate - Create faces and edges in the mesh
6291 
6292   Output Parameter:
6293 . dm  - The `DM` object representing the mesh
6294 
6295   Level: beginner
6296 
6297   Note:
6298   The format is the simplest possible:
6299 .vb
6300   dim Ne Nv Nc Nl
6301   v_1 v_2 ... v_Nc
6302   ...
6303   x y z marker_1 ... marker_Nl
6304 .ve
6305 
6306   Developer Note:
6307   Should use a `PetscViewer` not a filename
6308 
6309 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromFile()`, `DMPlexCreateGmsh()`, `DMPlexCreate()`
6310 */
6311 static PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
6312 {
6313   DMLabel      marker;
6314   PetscViewer  viewer;
6315   Vec          coordinates;
6316   PetscSection coordSection;
6317   PetscScalar *coords;
6318   char         line[PETSC_MAX_PATH_LEN];
6319   PetscInt     cdim, coordSize, v, c, d;
6320   PetscMPIInt  rank;
6321   int          snum, dim, Nv, Nc, Ncn, Nl;
6322 
6323   PetscFunctionBegin;
6324   PetscCallMPI(MPI_Comm_rank(comm, &rank));
6325   PetscCall(PetscViewerCreate(comm, &viewer));
6326   PetscCall(PetscViewerSetType(viewer, PETSCVIEWERASCII));
6327   PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
6328   PetscCall(PetscViewerFileSetName(viewer, filename));
6329   if (rank == 0) {
6330     PetscCall(PetscViewerRead(viewer, line, 5, NULL, PETSC_STRING));
6331     snum = sscanf(line, "%d %d %d %d %d", &dim, &Nc, &Nv, &Ncn, &Nl);
6332     PetscCheck(snum == 5, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
6333   } else {
6334     Nc = Nv = Ncn = Nl = 0;
6335   }
6336   PetscCallMPI(MPI_Bcast(&dim, 1, MPI_INT, 0, comm));
6337   cdim = dim;
6338   PetscCall(DMCreate(comm, dm));
6339   PetscCall(DMSetType(*dm, DMPLEX));
6340   PetscCall(DMPlexSetChart(*dm, 0, Nc + Nv));
6341   PetscCall(DMSetDimension(*dm, dim));
6342   PetscCall(DMSetCoordinateDim(*dm, cdim));
6343   /* Read topology */
6344   if (rank == 0) {
6345     char     format[PETSC_MAX_PATH_LEN];
6346     PetscInt cone[8];
6347     int      vbuf[8], v;
6348 
6349     for (c = 0; c < Ncn; ++c) {
6350       format[c * 3 + 0] = '%';
6351       format[c * 3 + 1] = 'd';
6352       format[c * 3 + 2] = ' ';
6353     }
6354     format[Ncn * 3 - 1] = '\0';
6355     for (c = 0; c < Nc; ++c) PetscCall(DMPlexSetConeSize(*dm, c, Ncn));
6356     PetscCall(DMSetUp(*dm));
6357     for (c = 0; c < Nc; ++c) {
6358       PetscCall(PetscViewerRead(viewer, line, Ncn, NULL, PETSC_STRING));
6359       switch (Ncn) {
6360       case 2:
6361         snum = sscanf(line, format, &vbuf[0], &vbuf[1]);
6362         break;
6363       case 3:
6364         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2]);
6365         break;
6366       case 4:
6367         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3]);
6368         break;
6369       case 6:
6370         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5]);
6371         break;
6372       case 8:
6373         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);
6374         break;
6375       default:
6376         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No cell shape with %d vertices", Ncn);
6377       }
6378       PetscCheck(snum == Ncn, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
6379       for (v = 0; v < Ncn; ++v) cone[v] = vbuf[v] + Nc;
6380       /* Hexahedra are inverted */
6381       if (Ncn == 8) {
6382         PetscInt tmp = cone[1];
6383         cone[1]      = cone[3];
6384         cone[3]      = tmp;
6385       }
6386       PetscCall(DMPlexSetCone(*dm, c, cone));
6387     }
6388   }
6389   PetscCall(DMPlexSymmetrize(*dm));
6390   PetscCall(DMPlexStratify(*dm));
6391   /* Read coordinates */
6392   PetscCall(DMGetCoordinateSection(*dm, &coordSection));
6393   PetscCall(PetscSectionSetNumFields(coordSection, 1));
6394   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
6395   PetscCall(PetscSectionSetChart(coordSection, Nc, Nc + Nv));
6396   for (v = Nc; v < Nc + Nv; ++v) {
6397     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
6398     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
6399   }
6400   PetscCall(PetscSectionSetUp(coordSection));
6401   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
6402   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
6403   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
6404   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
6405   PetscCall(VecSetBlockSize(coordinates, cdim));
6406   PetscCall(VecSetType(coordinates, VECSTANDARD));
6407   PetscCall(VecGetArray(coordinates, &coords));
6408   if (rank == 0) {
6409     char   format[PETSC_MAX_PATH_LEN];
6410     double x[3];
6411     int    l, val[3];
6412 
6413     if (Nl) {
6414       for (l = 0; l < Nl; ++l) {
6415         format[l * 3 + 0] = '%';
6416         format[l * 3 + 1] = 'd';
6417         format[l * 3 + 2] = ' ';
6418       }
6419       format[Nl * 3 - 1] = '\0';
6420       PetscCall(DMCreateLabel(*dm, "marker"));
6421       PetscCall(DMGetLabel(*dm, "marker", &marker));
6422     }
6423     for (v = 0; v < Nv; ++v) {
6424       PetscCall(PetscViewerRead(viewer, line, 3 + Nl, NULL, PETSC_STRING));
6425       snum = sscanf(line, "%lg %lg %lg", &x[0], &x[1], &x[2]);
6426       PetscCheck(snum == 3, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
6427       switch (Nl) {
6428       case 0:
6429         snum = 0;
6430         break;
6431       case 1:
6432         snum = sscanf(line, format, &val[0]);
6433         break;
6434       case 2:
6435         snum = sscanf(line, format, &val[0], &val[1]);
6436         break;
6437       case 3:
6438         snum = sscanf(line, format, &val[0], &val[1], &val[2]);
6439         break;
6440       default:
6441         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Request support for %d labels", Nl);
6442       }
6443       PetscCheck(snum == Nl, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
6444       for (d = 0; d < cdim; ++d) coords[v * cdim + d] = x[d];
6445       for (l = 0; l < Nl; ++l) PetscCall(DMLabelSetValue(marker, v + Nc, val[l]));
6446     }
6447   }
6448   PetscCall(VecRestoreArray(coordinates, &coords));
6449   PetscCall(DMSetCoordinatesLocal(*dm, coordinates));
6450   PetscCall(VecDestroy(&coordinates));
6451   PetscCall(PetscViewerDestroy(&viewer));
6452   if (interpolate) {
6453     DM      idm;
6454     DMLabel bdlabel;
6455 
6456     PetscCall(DMPlexInterpolate(*dm, &idm));
6457     PetscCall(DMDestroy(dm));
6458     *dm = idm;
6459 
6460     if (!Nl) {
6461       PetscCall(DMCreateLabel(*dm, "marker"));
6462       PetscCall(DMGetLabel(*dm, "marker", &bdlabel));
6463       PetscCall(DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel));
6464       PetscCall(DMPlexLabelComplete(*dm, bdlabel));
6465     }
6466   }
6467   PetscFunctionReturn(PETSC_SUCCESS);
6468 }
6469 
6470 /*@
6471   DMPlexCreateFromFile - This takes a filename and produces a `DM`
6472 
6473   Collective
6474 
6475   Input Parameters:
6476 + comm        - The communicator
6477 . filename    - A file name
6478 . plexname    - The object name of the resulting `DM`, also used for intra-datafile lookup by some formats
6479 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
6480 
6481   Output Parameter:
6482 . dm - The `DM`
6483 
6484   Options Database Key:
6485 . -dm_plex_create_from_hdf5_xdmf - use the `PETSC_VIEWER_HDF5_XDMF` format for reading HDF5
6486 
6487   Use `-dm_plex_create_ prefix` to pass options to the internal `PetscViewer`, e.g.
6488 $ -dm_plex_create_viewer_hdf5_collective
6489 
6490   Level: beginner
6491 
6492   Notes:
6493   Using `PETSCVIEWERHDF5` type with `PETSC_VIEWER_HDF5_PETSC` format, one can save multiple `DMPLEX`
6494   meshes in a single HDF5 file. This in turn requires one to name the `DMPLEX` object with `PetscObjectSetName()`
6495   before saving it with `DMView()` and before loading it with `DMLoad()` for identification of the mesh object.
6496   The input parameter name is thus used to name the `DMPLEX` object when `DMPlexCreateFromFile()` internally
6497   calls `DMLoad()`. Currently, name is ignored for other viewer types and/or formats.
6498 
6499 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`, `PetscObjectSetName()`, `DMView()`, `DMLoad()`
6500 @*/
6501 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], const char plexname[], PetscBool interpolate, DM *dm)
6502 {
6503   const char  extGmsh[]      = ".msh";
6504   const char  extGmsh2[]     = ".msh2";
6505   const char  extGmsh4[]     = ".msh4";
6506   const char  extCGNS[]      = ".cgns";
6507   const char  extExodus[]    = ".exo";
6508   const char  extExodus_e[]  = ".e";
6509   const char  extGenesis[]   = ".gen";
6510   const char  extFluent[]    = ".cas";
6511   const char  extHDF5[]      = ".h5";
6512   const char  extXDMFHDF5[]  = ".xdmf.h5";
6513   const char  extPLY[]       = ".ply";
6514   const char  extEGADSLite[] = ".egadslite";
6515   const char  extEGADS[]     = ".egads";
6516   const char  extIGES[]      = ".igs";
6517   const char  extSTEP[]      = ".stp";
6518   const char  extCV[]        = ".dat";
6519   size_t      len;
6520   PetscBool   isGmsh, isGmsh2, isGmsh4, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isPLY, isEGADSLite, isEGADS, isIGES, isSTEP, isCV, isXDMFHDF5;
6521   PetscMPIInt rank;
6522 
6523   PetscFunctionBegin;
6524   PetscAssertPointer(filename, 2);
6525   if (plexname) PetscAssertPointer(plexname, 3);
6526   PetscAssertPointer(dm, 5);
6527   PetscCall(DMInitializePackage());
6528   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromFile, 0, 0, 0, 0));
6529   PetscCallMPI(MPI_Comm_rank(comm, &rank));
6530   PetscCall(PetscStrlen(filename, &len));
6531   PetscCheck(len, comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
6532 
6533 #define CheckExtension(extension__, is_extension__) \
6534   do { \
6535     PetscAssert(sizeof(extension__), comm, PETSC_ERR_PLIB, "Zero-size extension: %s", extension__); \
6536     /* don't count the null-terminator at the end */ \
6537     const size_t ext_len = sizeof(extension__) - 1; \
6538     if (len < ext_len) { \
6539       is_extension__ = PETSC_FALSE; \
6540     } else { \
6541       PetscCall(PetscStrncmp(filename + len - ext_len, extension__, ext_len, &is_extension__)); \
6542     } \
6543   } while (0)
6544 
6545   CheckExtension(extGmsh, isGmsh);
6546   CheckExtension(extGmsh2, isGmsh2);
6547   CheckExtension(extGmsh4, isGmsh4);
6548   CheckExtension(extCGNS, isCGNS);
6549   CheckExtension(extExodus, isExodus);
6550   if (!isExodus) CheckExtension(extExodus_e, isExodus);
6551   CheckExtension(extGenesis, isGenesis);
6552   CheckExtension(extFluent, isFluent);
6553   CheckExtension(extHDF5, isHDF5);
6554   CheckExtension(extPLY, isPLY);
6555   CheckExtension(extEGADSLite, isEGADSLite);
6556   CheckExtension(extEGADS, isEGADS);
6557   CheckExtension(extIGES, isIGES);
6558   CheckExtension(extSTEP, isSTEP);
6559   CheckExtension(extCV, isCV);
6560   CheckExtension(extXDMFHDF5, isXDMFHDF5);
6561 
6562 #undef CheckExtension
6563 
6564   if (isGmsh || isGmsh2 || isGmsh4) {
6565     PetscCall(DMPlexCreateGmshFromFile(comm, filename, interpolate, dm));
6566   } else if (isCGNS) {
6567     PetscCall(DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm));
6568   } else if (isExodus || isGenesis) {
6569     PetscCall(DMPlexCreateExodusFromFile(comm, filename, interpolate, dm));
6570   } else if (isFluent) {
6571     PetscCall(DMPlexCreateFluentFromFile(comm, filename, interpolate, dm));
6572   } else if (isHDF5) {
6573     PetscViewer viewer;
6574 
6575     /* PETSC_VIEWER_HDF5_XDMF is used if the filename ends with .xdmf.h5, or if -dm_plex_create_from_hdf5_xdmf option is present */
6576     PetscCall(PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &isXDMFHDF5, NULL));
6577     PetscCall(PetscViewerCreate(comm, &viewer));
6578     PetscCall(PetscViewerSetType(viewer, PETSCVIEWERHDF5));
6579     PetscCall(PetscViewerSetOptionsPrefix(viewer, "dm_plex_create_"));
6580     PetscCall(PetscViewerSetFromOptions(viewer));
6581     PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
6582     PetscCall(PetscViewerFileSetName(viewer, filename));
6583 
6584     PetscCall(DMCreate(comm, dm));
6585     PetscCall(PetscObjectSetName((PetscObject)*dm, plexname));
6586     PetscCall(DMSetType(*dm, DMPLEX));
6587     if (isXDMFHDF5) PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF));
6588     PetscCall(DMLoad(*dm, viewer));
6589     if (isXDMFHDF5) PetscCall(PetscViewerPopFormat(viewer));
6590     PetscCall(PetscViewerDestroy(&viewer));
6591 
6592     if (interpolate) {
6593       DM idm;
6594 
6595       PetscCall(DMPlexInterpolate(*dm, &idm));
6596       PetscCall(DMDestroy(dm));
6597       *dm = idm;
6598     }
6599   } else if (isPLY) {
6600     PetscCall(DMPlexCreatePLYFromFile(comm, filename, interpolate, dm));
6601   } else if (isEGADSLite || isEGADS || isIGES || isSTEP) {
6602     if (isEGADSLite) PetscCall(DMPlexCreateEGADSLiteFromFile(comm, filename, dm));
6603     else PetscCall(DMPlexCreateEGADSFromFile(comm, filename, dm));
6604     if (!interpolate) {
6605       DM udm;
6606 
6607       PetscCall(DMPlexUninterpolate(*dm, &udm));
6608       PetscCall(DMDestroy(dm));
6609       *dm = udm;
6610     }
6611   } else if (isCV) {
6612     PetscCall(DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm));
6613   } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
6614   PetscCall(PetscStrlen(plexname, &len));
6615   if (len) PetscCall(PetscObjectSetName((PetscObject)*dm, plexname));
6616   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromFile, 0, 0, 0, 0));
6617   PetscFunctionReturn(PETSC_SUCCESS);
6618 }
6619 
6620 /*@
6621   DMPlexCreateEphemeral - This takes a `DMPlexTransform` and a base `DMPlex` and produces an ephemeral `DM`, meaning one that is created on the fly in response to queries.
6622 
6623   Input Parameters:
6624 + tr     - The `DMPlexTransform`
6625 - prefix - An options prefix, or NULL
6626 
6627   Output Parameter:
6628 . dm - The `DM`
6629 
6630   Level: beginner
6631 
6632   Notes:
6633   An emphemeral mesh is one that is not stored concretely, as in the default `DMPLEX` implementation, but rather is produced on the fly in response to queries, using information from the transform and the base mesh.
6634 
6635 .seealso: `DMPlexCreateFromFile`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
6636 @*/
6637 PetscErrorCode DMPlexCreateEphemeral(DMPlexTransform tr, const char prefix[], DM *dm)
6638 {
6639   DM           bdm, bcdm, cdm;
6640   Vec          coordinates, coordinatesNew;
6641   PetscSection cs;
6642   PetscInt     cdim, Nl;
6643 
6644   PetscFunctionBegin;
6645   PetscCall(DMCreate(PetscObjectComm((PetscObject)tr), dm));
6646   PetscCall(DMSetType(*dm, DMPLEX));
6647   ((DM_Plex *)(*dm)->data)->interpolated = DMPLEX_INTERPOLATED_FULL;
6648   // Handle coordinates
6649   PetscCall(DMPlexTransformGetDM(tr, &bdm));
6650   PetscCall(DMPlexTransformSetDimensions(tr, bdm, *dm));
6651   PetscCall(DMGetCoordinateDim(*dm, &cdim));
6652   PetscCall(DMGetCoordinateDM(bdm, &bcdm));
6653   PetscCall(DMGetCoordinateDM(*dm, &cdm));
6654   PetscCall(DMCopyDisc(bcdm, cdm));
6655   PetscCall(DMGetLocalSection(cdm, &cs));
6656   PetscCall(PetscSectionSetNumFields(cs, 1));
6657   PetscCall(PetscSectionSetFieldComponents(cs, 0, cdim));
6658   PetscCall(DMGetCoordinatesLocal(bdm, &coordinates));
6659   PetscCall(VecDuplicate(coordinates, &coordinatesNew));
6660   PetscCall(VecCopy(coordinates, coordinatesNew));
6661   PetscCall(DMSetCoordinatesLocal(*dm, coordinatesNew));
6662   PetscCall(VecDestroy(&coordinatesNew));
6663 
6664   PetscCall(PetscObjectReference((PetscObject)tr));
6665   PetscCall(DMPlexTransformDestroy(&((DM_Plex *)(*dm)->data)->tr));
6666   ((DM_Plex *)(*dm)->data)->tr = tr;
6667   PetscCall(DMPlexDistributeSetDefault(*dm, PETSC_FALSE));
6668   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)*dm, prefix));
6669   PetscCall(DMSetFromOptions(*dm));
6670 
6671   PetscCall(DMGetNumLabels(bdm, &Nl));
6672   for (PetscInt l = 0; l < Nl; ++l) {
6673     DMLabel     label, labelNew;
6674     const char *lname;
6675     PetscBool   isDepth, isCellType;
6676 
6677     PetscCall(DMGetLabelName(bdm, l, &lname));
6678     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
6679     if (isDepth) continue;
6680     PetscCall(PetscStrcmp(lname, "celltype", &isCellType));
6681     if (isCellType) continue;
6682     PetscCall(DMCreateLabel(*dm, lname));
6683     PetscCall(DMGetLabel(bdm, lname, &label));
6684     PetscCall(DMGetLabel(*dm, lname, &labelNew));
6685     PetscCall(DMLabelSetType(labelNew, DMLABELEPHEMERAL));
6686     PetscCall(DMLabelEphemeralSetLabel(labelNew, label));
6687     PetscCall(DMLabelEphemeralSetTransform(labelNew, tr));
6688     PetscCall(DMLabelSetUp(labelNew));
6689   }
6690   PetscFunctionReturn(PETSC_SUCCESS);
6691 }
6692