xref: /petsc/src/dm/impls/plex/plexcreate.c (revision 476787b73747423b72d8c77dc2f682bd229b3ce2)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h> /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>   /*I   "petscdmplex.h"   I*/
4 #include <petscsf.h>
5 #include <petscdmplextransform.h>
6 #include <petscdmlabelephemeral.h>
7 #include <petsc/private/kernels/blockmatmult.h>
8 #include <petsc/private/kernels/blockinvert.h>
9 
10 PetscLogEvent DMPLEX_CreateFromFile, DMPLEX_CreateFromOptions, DMPLEX_BuildFromCellList, DMPLEX_BuildCoordinatesFromCellList;
11 
12 /* External function declarations here */
13 static PetscErrorCode DMInitialize_Plex(DM dm);
14 
15 /* This copies internal things in the Plex structure that we generally want when making a new, related Plex */
16 PetscErrorCode DMPlexCopy_Internal(DM dmin, PetscBool copyPeriodicity, PetscBool copyOverlap, DM dmout)
17 {
18   const PetscReal         *maxCell, *Lstart, *L;
19   PetscBool                dist, useCeed;
20   DMPlexReorderDefaultFlag reorder;
21 
22   PetscFunctionBegin;
23   if (copyPeriodicity) {
24     PetscCall(DMGetPeriodicity(dmin, &maxCell, &Lstart, &L));
25     PetscCall(DMSetPeriodicity(dmout, maxCell, Lstart, L));
26   }
27   PetscCall(DMPlexDistributeGetDefault(dmin, &dist));
28   PetscCall(DMPlexDistributeSetDefault(dmout, dist));
29   PetscCall(DMPlexReorderGetDefault(dmin, &reorder));
30   PetscCall(DMPlexReorderSetDefault(dmout, reorder));
31   PetscCall(DMPlexGetUseCeed(dmin, &useCeed));
32   PetscCall(DMPlexSetUseCeed(dmout, useCeed));
33   ((DM_Plex *)dmout->data)->useHashLocation = ((DM_Plex *)dmin->data)->useHashLocation;
34   ((DM_Plex *)dmout->data)->printSetValues  = ((DM_Plex *)dmin->data)->printSetValues;
35   ((DM_Plex *)dmout->data)->printFEM        = ((DM_Plex *)dmin->data)->printFEM;
36   ((DM_Plex *)dmout->data)->printFVM        = ((DM_Plex *)dmin->data)->printFVM;
37   ((DM_Plex *)dmout->data)->printL2         = ((DM_Plex *)dmin->data)->printL2;
38   ((DM_Plex *)dmout->data)->printLocate     = ((DM_Plex *)dmin->data)->printLocate;
39   ((DM_Plex *)dmout->data)->printTol        = ((DM_Plex *)dmin->data)->printTol;
40   if (copyOverlap) PetscCall(DMPlexSetOverlap_Plex(dmout, dmin, 0));
41   PetscFunctionReturn(PETSC_SUCCESS);
42 }
43 
44 /* Replace dm with the contents of ndm, and then destroy ndm
45    - Share the DM_Plex structure
46    - Share the coordinates
47    - Share the SF
48 */
49 PetscErrorCode DMPlexReplace_Internal(DM dm, DM *ndm)
50 {
51   PetscSF          sf;
52   DM               dmNew = *ndm, coordDM, coarseDM;
53   Vec              coords;
54   const PetscReal *maxCell, *Lstart, *L;
55   PetscInt         dim, cdim;
56 
57   PetscFunctionBegin;
58   if (dm == dmNew) {
59     PetscCall(DMDestroy(ndm));
60     PetscFunctionReturn(PETSC_SUCCESS);
61   }
62   dm->setupcalled = dmNew->setupcalled;
63   PetscCall(DMGetDimension(dmNew, &dim));
64   PetscCall(DMSetDimension(dm, dim));
65   PetscCall(DMGetCoordinateDim(dmNew, &cdim));
66   PetscCall(DMSetCoordinateDim(dm, cdim));
67   PetscCall(DMGetPointSF(dmNew, &sf));
68   PetscCall(DMSetPointSF(dm, sf));
69   PetscCall(DMGetCoordinateDM(dmNew, &coordDM));
70   PetscCall(DMGetCoordinatesLocal(dmNew, &coords));
71   PetscCall(DMSetCoordinateDM(dm, coordDM));
72   PetscCall(DMSetCoordinatesLocal(dm, coords));
73   PetscCall(DMGetCellCoordinateDM(dmNew, &coordDM));
74   PetscCall(DMGetCellCoordinatesLocal(dmNew, &coords));
75   PetscCall(DMSetCellCoordinateDM(dm, coordDM));
76   PetscCall(DMSetCellCoordinatesLocal(dm, coords));
77   /* Do not want to create the coordinate field if it does not already exist, so do not call DMGetCoordinateField() */
78   PetscCall(DMFieldDestroy(&dm->coordinates[0].field));
79   dm->coordinates[0].field            = dmNew->coordinates[0].field;
80   ((DM_Plex *)dmNew->data)->coordFunc = ((DM_Plex *)dm->data)->coordFunc;
81   PetscCall(DMGetPeriodicity(dmNew, &maxCell, &Lstart, &L));
82   PetscCall(DMSetPeriodicity(dm, maxCell, Lstart, L));
83   PetscCall(DMPlexGetGlobalToNaturalSF(dmNew, &sf));
84   PetscCall(DMPlexSetGlobalToNaturalSF(dm, sf));
85   PetscCall(DMDestroy_Plex(dm));
86   PetscCall(DMInitialize_Plex(dm));
87   dm->data = dmNew->data;
88   ((DM_Plex *)dmNew->data)->refct++;
89   PetscCall(DMPlexGetIsoperiodicFaceSF(dm, &sf));
90   PetscCall(DMPlexSetIsoperiodicFaceSF(dm, sf)); // for the compose function effect on dm
91   PetscCall(DMDestroyLabelLinkList_Internal(dm));
92   PetscCall(DMCopyLabels(dmNew, dm, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL));
93   PetscCall(DMGetCoarseDM(dmNew, &coarseDM));
94   PetscCall(DMSetCoarseDM(dm, coarseDM));
95   PetscCall(DMDestroy(ndm));
96   PetscFunctionReturn(PETSC_SUCCESS);
97 }
98 
99 /* Swap dm with the contents of dmNew
100    - Swap the DM_Plex structure
101    - Swap the coordinates
102    - Swap the point PetscSF
103 */
104 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
105 {
106   DM          coordDMA, coordDMB;
107   Vec         coordsA, coordsB;
108   PetscSF     sfA, sfB;
109   DMField     fieldTmp;
110   void       *tmp;
111   DMLabelLink listTmp;
112   DMLabel     depthTmp;
113   PetscInt    tmpI;
114 
115   PetscFunctionBegin;
116   if (dmA == dmB) PetscFunctionReturn(PETSC_SUCCESS);
117   PetscCall(DMGetPointSF(dmA, &sfA));
118   PetscCall(DMGetPointSF(dmB, &sfB));
119   PetscCall(PetscObjectReference((PetscObject)sfA));
120   PetscCall(DMSetPointSF(dmA, sfB));
121   PetscCall(DMSetPointSF(dmB, sfA));
122   PetscCall(PetscObjectDereference((PetscObject)sfA));
123 
124   PetscCall(DMGetCoordinateDM(dmA, &coordDMA));
125   PetscCall(DMGetCoordinateDM(dmB, &coordDMB));
126   PetscCall(PetscObjectReference((PetscObject)coordDMA));
127   PetscCall(DMSetCoordinateDM(dmA, coordDMB));
128   PetscCall(DMSetCoordinateDM(dmB, coordDMA));
129   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
130 
131   PetscCall(DMGetCoordinatesLocal(dmA, &coordsA));
132   PetscCall(DMGetCoordinatesLocal(dmB, &coordsB));
133   PetscCall(PetscObjectReference((PetscObject)coordsA));
134   PetscCall(DMSetCoordinatesLocal(dmA, coordsB));
135   PetscCall(DMSetCoordinatesLocal(dmB, coordsA));
136   PetscCall(PetscObjectDereference((PetscObject)coordsA));
137 
138   PetscCall(DMGetCellCoordinateDM(dmA, &coordDMA));
139   PetscCall(DMGetCellCoordinateDM(dmB, &coordDMB));
140   PetscCall(PetscObjectReference((PetscObject)coordDMA));
141   PetscCall(DMSetCellCoordinateDM(dmA, coordDMB));
142   PetscCall(DMSetCellCoordinateDM(dmB, coordDMA));
143   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
144 
145   PetscCall(DMGetCellCoordinatesLocal(dmA, &coordsA));
146   PetscCall(DMGetCellCoordinatesLocal(dmB, &coordsB));
147   PetscCall(PetscObjectReference((PetscObject)coordsA));
148   PetscCall(DMSetCellCoordinatesLocal(dmA, coordsB));
149   PetscCall(DMSetCellCoordinatesLocal(dmB, coordsA));
150   PetscCall(PetscObjectDereference((PetscObject)coordsA));
151 
152   fieldTmp                  = dmA->coordinates[0].field;
153   dmA->coordinates[0].field = dmB->coordinates[0].field;
154   dmB->coordinates[0].field = fieldTmp;
155   fieldTmp                  = dmA->coordinates[1].field;
156   dmA->coordinates[1].field = dmB->coordinates[1].field;
157   dmB->coordinates[1].field = fieldTmp;
158   tmp                       = dmA->data;
159   dmA->data                 = dmB->data;
160   dmB->data                 = tmp;
161   listTmp                   = dmA->labels;
162   dmA->labels               = dmB->labels;
163   dmB->labels               = listTmp;
164   depthTmp                  = dmA->depthLabel;
165   dmA->depthLabel           = dmB->depthLabel;
166   dmB->depthLabel           = depthTmp;
167   depthTmp                  = dmA->celltypeLabel;
168   dmA->celltypeLabel        = dmB->celltypeLabel;
169   dmB->celltypeLabel        = depthTmp;
170   tmpI                      = dmA->levelup;
171   dmA->levelup              = dmB->levelup;
172   dmB->levelup              = tmpI;
173   PetscFunctionReturn(PETSC_SUCCESS);
174 }
175 
176 PetscErrorCode DMPlexInterpolateInPlace_Internal(DM dm)
177 {
178   DM idm;
179 
180   PetscFunctionBegin;
181   PetscCall(DMPlexInterpolate(dm, &idm));
182   PetscCall(DMPlexCopyCoordinates(dm, idm));
183   PetscCall(DMPlexReplace_Internal(dm, &idm));
184   PetscFunctionReturn(PETSC_SUCCESS);
185 }
186 
187 /*@C
188   DMPlexCreateCoordinateSpace - Creates a finite element space for the coordinates
189 
190   Collective
191 
192   Input Parameters:
193 + dm        - The `DMPLEX`
194 . degree    - The degree of the finite element or `PETSC_DECIDE`
195 . project   - Flag to project current coordinates into the space
196 - coordFunc - An optional function to map new points from refinement to the surface
197 
198   Level: advanced
199 
200 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `PetscPointFunc`, `PetscFECreateLagrange()`, `DMGetCoordinateDM()`
201 @*/
202 PetscErrorCode DMPlexCreateCoordinateSpace(DM dm, PetscInt degree, PetscBool project, PetscPointFunc coordFunc)
203 {
204   DM_Plex *mesh = (DM_Plex *)dm->data;
205   PetscFE  fe   = NULL;
206   DM       cdm;
207   PetscInt dim, dE, qorder;
208 
209   PetscFunctionBegin;
210   PetscCall(DMGetDimension(dm, &dim));
211   PetscCall(DMGetCoordinateDim(dm, &dE));
212   qorder = degree;
213   PetscCall(DMGetCoordinateDM(dm, &cdm));
214   PetscObjectOptionsBegin((PetscObject)cdm);
215   PetscCall(PetscOptionsBoundedInt("-default_quadrature_order", "Quadrature order is one less than quadrature points per edge", "DMPlexCreateCoordinateSpace", qorder, &qorder, NULL, 0));
216   PetscOptionsEnd();
217   if (degree >= 0) {
218     DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
219     PetscInt       cStart, cEnd, gct;
220 
221     PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
222     if (cEnd > cStart) PetscCall(DMPlexGetCellType(dm, cStart, &ct));
223     gct = (PetscInt)ct;
224     PetscCall(MPIU_Allreduce(MPI_IN_PLACE, &gct, 1, MPIU_INT, MPI_MIN, PetscObjectComm((PetscObject)dm)));
225     ct = (DMPolytopeType)gct;
226     // Work around current bug in PetscDualSpaceSetUp_Lagrange()
227     //   Can be seen in plex_tutorials-ex10_1
228     if (ct != DM_POLYTOPE_SEG_PRISM_TENSOR && ct != DM_POLYTOPE_TRI_PRISM_TENSOR && ct != DM_POLYTOPE_QUAD_PRISM_TENSOR) PetscCall(PetscFECreateLagrangeByCell(PETSC_COMM_SELF, dim, dE, ct, degree, qorder, &fe));
229   }
230   PetscCall(DMSetCoordinateDisc(dm, fe, project));
231   PetscCall(PetscFEDestroy(&fe));
232   mesh->coordFunc = coordFunc;
233   PetscFunctionReturn(PETSC_SUCCESS);
234 }
235 
236 /*@
237   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
238 
239   Collective
240 
241   Input Parameters:
242 + comm            - The communicator for the `DM` object
243 . dim             - The spatial dimension
244 . simplex         - Flag for simplicial cells, otherwise they are tensor product cells
245 . interpolate     - Flag to create intermediate mesh pieces (edges, faces)
246 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
247 
248   Output Parameter:
249 . newdm - The `DM` object
250 
251   Level: beginner
252 
253 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetType()`, `DMCreate()`
254 @*/
255 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscReal refinementLimit, DM *newdm)
256 {
257   DM          dm;
258   PetscMPIInt rank;
259 
260   PetscFunctionBegin;
261   PetscCall(DMCreate(comm, &dm));
262   PetscCall(DMSetType(dm, DMPLEX));
263   PetscCall(DMSetDimension(dm, dim));
264   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
265   PetscCallMPI(MPI_Comm_rank(comm, &rank));
266   switch (dim) {
267   case 2:
268     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "triangular"));
269     else PetscCall(PetscObjectSetName((PetscObject)dm, "quadrilateral"));
270     break;
271   case 3:
272     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "tetrahedral"));
273     else PetscCall(PetscObjectSetName((PetscObject)dm, "hexahedral"));
274     break;
275   default:
276     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
277   }
278   if (rank) {
279     PetscInt numPoints[2] = {0, 0};
280     PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL));
281   } else {
282     switch (dim) {
283     case 2:
284       if (simplex) {
285         PetscInt    numPoints[2]        = {4, 2};
286         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
287         PetscInt    cones[6]            = {2, 3, 4, 5, 4, 3};
288         PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
289         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
290 
291         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
292       } else {
293         PetscInt    numPoints[2]        = {6, 2};
294         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
295         PetscInt    cones[8]            = {2, 3, 4, 5, 3, 6, 7, 4};
296         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
297         PetscScalar vertexCoords[12]    = {-1.0, -0.5, 0.0, -0.5, 0.0, 0.5, -1.0, 0.5, 1.0, -0.5, 1.0, 0.5};
298 
299         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
300       }
301       break;
302     case 3:
303       if (simplex) {
304         PetscInt    numPoints[2]        = {5, 2};
305         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
306         PetscInt    cones[8]            = {4, 3, 5, 2, 5, 3, 4, 6};
307         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
308         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0};
309 
310         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
311       } else {
312         PetscInt    numPoints[2]         = {12, 2};
313         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
314         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9, 5, 4, 10, 11, 7, 12, 13, 8};
315         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
316         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5, -1.0, 0.5, -0.5, 0.0, 0.5, -0.5, 0.0, -0.5, -0.5, -1.0, -0.5, 0.5, 0.0, -0.5, 0.5, 0.0, 0.5, 0.5, -1.0, 0.5, 0.5, 1.0, 0.5, -0.5, 1.0, -0.5, -0.5, 1.0, -0.5, 0.5, 1.0, 0.5, 0.5};
317 
318         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
319       }
320       break;
321     default:
322       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
323     }
324   }
325   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
326   *newdm = dm;
327   if (refinementLimit > 0.0) {
328     DM          rdm;
329     const char *name;
330 
331     PetscCall(DMPlexSetRefinementUniform(*newdm, PETSC_FALSE));
332     PetscCall(DMPlexSetRefinementLimit(*newdm, refinementLimit));
333     PetscCall(DMRefine(*newdm, comm, &rdm));
334     PetscCall(PetscObjectGetName((PetscObject)*newdm, &name));
335     PetscCall(PetscObjectSetName((PetscObject)rdm, name));
336     PetscCall(DMDestroy(newdm));
337     *newdm = rdm;
338   }
339   if (interpolate) {
340     DM idm;
341 
342     PetscCall(DMPlexInterpolate(*newdm, &idm));
343     PetscCall(DMDestroy(newdm));
344     *newdm = idm;
345   }
346   PetscFunctionReturn(PETSC_SUCCESS);
347 }
348 
349 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
350 {
351   const PetscInt numVertices    = 2;
352   PetscInt       markerRight    = 1;
353   PetscInt       markerLeft     = 1;
354   PetscBool      markerSeparate = PETSC_FALSE;
355   Vec            coordinates;
356   PetscSection   coordSection;
357   PetscScalar   *coords;
358   PetscInt       coordSize;
359   PetscMPIInt    rank;
360   PetscInt       cdim = 1, v;
361 
362   PetscFunctionBegin;
363   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
364   if (markerSeparate) {
365     markerRight = 2;
366     markerLeft  = 1;
367   }
368   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
369   if (rank == 0) {
370     PetscCall(DMPlexSetChart(dm, 0, numVertices));
371     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
372     PetscCall(DMSetLabelValue(dm, "marker", 0, markerLeft));
373     PetscCall(DMSetLabelValue(dm, "marker", 1, markerRight));
374   }
375   PetscCall(DMPlexSymmetrize(dm));
376   PetscCall(DMPlexStratify(dm));
377   /* Build coordinates */
378   PetscCall(DMSetCoordinateDim(dm, cdim));
379   PetscCall(DMGetCoordinateSection(dm, &coordSection));
380   PetscCall(PetscSectionSetNumFields(coordSection, 1));
381   PetscCall(PetscSectionSetChart(coordSection, 0, numVertices));
382   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
383   for (v = 0; v < numVertices; ++v) {
384     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
385     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
386   }
387   PetscCall(PetscSectionSetUp(coordSection));
388   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
389   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
390   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
391   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
392   PetscCall(VecSetBlockSize(coordinates, cdim));
393   PetscCall(VecSetType(coordinates, VECSTANDARD));
394   PetscCall(VecGetArray(coordinates, &coords));
395   coords[0] = lower[0];
396   coords[1] = upper[0];
397   PetscCall(VecRestoreArray(coordinates, &coords));
398   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
399   PetscCall(VecDestroy(&coordinates));
400   PetscFunctionReturn(PETSC_SUCCESS);
401 }
402 
403 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
404 {
405   const PetscInt numVertices    = (edges[0] + 1) * (edges[1] + 1);
406   const PetscInt numEdges       = edges[0] * (edges[1] + 1) + (edges[0] + 1) * edges[1];
407   PetscInt       markerTop      = 1;
408   PetscInt       markerBottom   = 1;
409   PetscInt       markerRight    = 1;
410   PetscInt       markerLeft     = 1;
411   PetscBool      markerSeparate = PETSC_FALSE;
412   Vec            coordinates;
413   PetscSection   coordSection;
414   PetscScalar   *coords;
415   PetscInt       coordSize;
416   PetscMPIInt    rank;
417   PetscInt       v, vx, vy;
418 
419   PetscFunctionBegin;
420   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
421   if (markerSeparate) {
422     markerTop    = 3;
423     markerBottom = 1;
424     markerRight  = 2;
425     markerLeft   = 4;
426   }
427   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
428   if (rank == 0) {
429     PetscInt e, ex, ey;
430 
431     PetscCall(DMPlexSetChart(dm, 0, numEdges + numVertices));
432     for (e = 0; e < numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
433     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
434     for (vx = 0; vx <= edges[0]; vx++) {
435       for (ey = 0; ey < edges[1]; ey++) {
436         PetscInt edge   = vx * edges[1] + ey + edges[0] * (edges[1] + 1);
437         PetscInt vertex = ey * (edges[0] + 1) + vx + numEdges;
438         PetscInt cone[2];
439 
440         cone[0] = vertex;
441         cone[1] = vertex + edges[0] + 1;
442         PetscCall(DMPlexSetCone(dm, edge, cone));
443         if (vx == edges[0]) {
444           PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
445           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
446           if (ey == edges[1] - 1) {
447             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
448             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerRight));
449           }
450         } else if (vx == 0) {
451           PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
452           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
453           if (ey == edges[1] - 1) {
454             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
455             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft));
456           }
457         }
458       }
459     }
460     for (vy = 0; vy <= edges[1]; vy++) {
461       for (ex = 0; ex < edges[0]; ex++) {
462         PetscInt edge   = vy * edges[0] + ex;
463         PetscInt vertex = vy * (edges[0] + 1) + ex + numEdges;
464         PetscInt cone[2];
465 
466         cone[0] = vertex;
467         cone[1] = vertex + 1;
468         PetscCall(DMPlexSetCone(dm, edge, cone));
469         if (vy == edges[1]) {
470           PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
471           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
472           if (ex == edges[0] - 1) {
473             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
474             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerTop));
475           }
476         } else if (vy == 0) {
477           PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
478           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
479           if (ex == edges[0] - 1) {
480             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
481             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom));
482           }
483         }
484       }
485     }
486   }
487   PetscCall(DMPlexSymmetrize(dm));
488   PetscCall(DMPlexStratify(dm));
489   /* Build coordinates */
490   PetscCall(DMSetCoordinateDim(dm, 2));
491   PetscCall(DMGetCoordinateSection(dm, &coordSection));
492   PetscCall(PetscSectionSetNumFields(coordSection, 1));
493   PetscCall(PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices));
494   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 2));
495   for (v = numEdges; v < numEdges + numVertices; ++v) {
496     PetscCall(PetscSectionSetDof(coordSection, v, 2));
497     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 2));
498   }
499   PetscCall(PetscSectionSetUp(coordSection));
500   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
501   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
502   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
503   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
504   PetscCall(VecSetBlockSize(coordinates, 2));
505   PetscCall(VecSetType(coordinates, VECSTANDARD));
506   PetscCall(VecGetArray(coordinates, &coords));
507   for (vy = 0; vy <= edges[1]; ++vy) {
508     for (vx = 0; vx <= edges[0]; ++vx) {
509       coords[(vy * (edges[0] + 1) + vx) * 2 + 0] = lower[0] + ((upper[0] - lower[0]) / edges[0]) * vx;
510       coords[(vy * (edges[0] + 1) + vx) * 2 + 1] = lower[1] + ((upper[1] - lower[1]) / edges[1]) * vy;
511     }
512   }
513   PetscCall(VecRestoreArray(coordinates, &coords));
514   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
515   PetscCall(VecDestroy(&coordinates));
516   PetscFunctionReturn(PETSC_SUCCESS);
517 }
518 
519 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
520 {
521   PetscInt     vertices[3], numVertices;
522   PetscInt     numFaces       = 2 * faces[0] * faces[1] + 2 * faces[1] * faces[2] + 2 * faces[0] * faces[2];
523   PetscInt     markerTop      = 1;
524   PetscInt     markerBottom   = 1;
525   PetscInt     markerFront    = 1;
526   PetscInt     markerBack     = 1;
527   PetscInt     markerRight    = 1;
528   PetscInt     markerLeft     = 1;
529   PetscBool    markerSeparate = PETSC_FALSE;
530   Vec          coordinates;
531   PetscSection coordSection;
532   PetscScalar *coords;
533   PetscInt     coordSize;
534   PetscMPIInt  rank;
535   PetscInt     v, vx, vy, vz;
536   PetscInt     voffset, iface = 0, cone[4];
537 
538   PetscFunctionBegin;
539   PetscCheck(faces[0] >= 1 && faces[1] >= 1 && faces[2] >= 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
540   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
541   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
542   if (markerSeparate) {
543     markerBottom = 1;
544     markerTop    = 2;
545     markerFront  = 3;
546     markerBack   = 4;
547     markerRight  = 5;
548     markerLeft   = 6;
549   }
550   vertices[0] = faces[0] + 1;
551   vertices[1] = faces[1] + 1;
552   vertices[2] = faces[2] + 1;
553   numVertices = vertices[0] * vertices[1] * vertices[2];
554   if (rank == 0) {
555     PetscInt f;
556 
557     PetscCall(DMPlexSetChart(dm, 0, numFaces + numVertices));
558     for (f = 0; f < numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
559     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
560 
561     /* Side 0 (Top) */
562     for (vy = 0; vy < faces[1]; vy++) {
563       for (vx = 0; vx < faces[0]; vx++) {
564         voffset = numFaces + vertices[0] * vertices[1] * (vertices[2] - 1) + vy * vertices[0] + vx;
565         cone[0] = voffset;
566         cone[1] = voffset + 1;
567         cone[2] = voffset + vertices[0] + 1;
568         cone[3] = voffset + vertices[0];
569         PetscCall(DMPlexSetCone(dm, iface, cone));
570         PetscCall(DMSetLabelValue(dm, "marker", iface, markerTop));
571         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerTop));
572         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerTop));
573         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerTop));
574         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerTop));
575         iface++;
576       }
577     }
578 
579     /* Side 1 (Bottom) */
580     for (vy = 0; vy < faces[1]; vy++) {
581       for (vx = 0; vx < faces[0]; vx++) {
582         voffset = numFaces + vy * (faces[0] + 1) + vx;
583         cone[0] = voffset + 1;
584         cone[1] = voffset;
585         cone[2] = voffset + vertices[0];
586         cone[3] = voffset + vertices[0] + 1;
587         PetscCall(DMPlexSetCone(dm, iface, cone));
588         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBottom));
589         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBottom));
590         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBottom));
591         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerBottom));
592         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerBottom));
593         iface++;
594       }
595     }
596 
597     /* Side 2 (Front) */
598     for (vz = 0; vz < faces[2]; vz++) {
599       for (vx = 0; vx < faces[0]; vx++) {
600         voffset = numFaces + vz * vertices[0] * vertices[1] + vx;
601         cone[0] = voffset;
602         cone[1] = voffset + 1;
603         cone[2] = voffset + vertices[0] * vertices[1] + 1;
604         cone[3] = voffset + vertices[0] * vertices[1];
605         PetscCall(DMPlexSetCone(dm, iface, cone));
606         PetscCall(DMSetLabelValue(dm, "marker", iface, markerFront));
607         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerFront));
608         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerFront));
609         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerFront));
610         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerFront));
611         iface++;
612       }
613     }
614 
615     /* Side 3 (Back) */
616     for (vz = 0; vz < faces[2]; vz++) {
617       for (vx = 0; vx < faces[0]; vx++) {
618         voffset = numFaces + vz * vertices[0] * vertices[1] + vertices[0] * (vertices[1] - 1) + vx;
619         cone[0] = voffset + vertices[0] * vertices[1];
620         cone[1] = voffset + vertices[0] * vertices[1] + 1;
621         cone[2] = voffset + 1;
622         cone[3] = voffset;
623         PetscCall(DMPlexSetCone(dm, iface, cone));
624         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBack));
625         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBack));
626         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBack));
627         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerBack));
628         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerBack));
629         iface++;
630       }
631     }
632 
633     /* Side 4 (Left) */
634     for (vz = 0; vz < faces[2]; vz++) {
635       for (vy = 0; vy < faces[1]; vy++) {
636         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0];
637         cone[0] = voffset;
638         cone[1] = voffset + vertices[0] * vertices[1];
639         cone[2] = voffset + vertices[0] * vertices[1] + vertices[0];
640         cone[3] = voffset + vertices[0];
641         PetscCall(DMPlexSetCone(dm, iface, cone));
642         PetscCall(DMSetLabelValue(dm, "marker", iface, markerLeft));
643         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerLeft));
644         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerLeft));
645         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[1] + 0, markerLeft));
646         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerLeft));
647         iface++;
648       }
649     }
650 
651     /* Side 5 (Right) */
652     for (vz = 0; vz < faces[2]; vz++) {
653       for (vy = 0; vy < faces[1]; vy++) {
654         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0] + faces[0];
655         cone[0] = voffset + vertices[0] * vertices[1];
656         cone[1] = voffset;
657         cone[2] = voffset + vertices[0];
658         cone[3] = voffset + vertices[0] * vertices[1] + vertices[0];
659         PetscCall(DMPlexSetCone(dm, iface, cone));
660         PetscCall(DMSetLabelValue(dm, "marker", iface, markerRight));
661         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerRight));
662         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerRight));
663         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerRight));
664         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerRight));
665         iface++;
666       }
667     }
668   }
669   PetscCall(DMPlexSymmetrize(dm));
670   PetscCall(DMPlexStratify(dm));
671   /* Build coordinates */
672   PetscCall(DMSetCoordinateDim(dm, 3));
673   PetscCall(DMGetCoordinateSection(dm, &coordSection));
674   PetscCall(PetscSectionSetNumFields(coordSection, 1));
675   PetscCall(PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices));
676   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 3));
677   for (v = numFaces; v < numFaces + numVertices; ++v) {
678     PetscCall(PetscSectionSetDof(coordSection, v, 3));
679     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 3));
680   }
681   PetscCall(PetscSectionSetUp(coordSection));
682   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
683   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
684   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
685   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
686   PetscCall(VecSetBlockSize(coordinates, 3));
687   PetscCall(VecSetType(coordinates, VECSTANDARD));
688   PetscCall(VecGetArray(coordinates, &coords));
689   for (vz = 0; vz <= faces[2]; ++vz) {
690     for (vy = 0; vy <= faces[1]; ++vy) {
691       for (vx = 0; vx <= faces[0]; ++vx) {
692         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 0] = lower[0] + ((upper[0] - lower[0]) / faces[0]) * vx;
693         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 1] = lower[1] + ((upper[1] - lower[1]) / faces[1]) * vy;
694         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 2] = lower[2] + ((upper[2] - lower[2]) / faces[2]) * vz;
695       }
696     }
697   }
698   PetscCall(VecRestoreArray(coordinates, &coords));
699   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
700   PetscCall(VecDestroy(&coordinates));
701   PetscFunctionReturn(PETSC_SUCCESS);
702 }
703 
704 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate)
705 {
706   PetscFunctionBegin;
707   PetscValidLogicalCollectiveInt(dm, dim, 2);
708   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
709   PetscCall(DMSetDimension(dm, dim - 1));
710   PetscCall(DMSetCoordinateDim(dm, dim));
711   switch (dim) {
712   case 1:
713     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(dm, lower, upper, faces));
714     break;
715   case 2:
716     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(dm, lower, upper, faces));
717     break;
718   case 3:
719     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(dm, lower, upper, faces));
720     break;
721   default:
722     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Dimension not supported: %" PetscInt_FMT, dim);
723   }
724   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
725   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
726   PetscFunctionReturn(PETSC_SUCCESS);
727 }
728 
729 /*@C
730   DMPlexCreateBoxSurfaceMesh - Creates a mesh on the surface of the tensor product of unit intervals (box) using tensor cells (hexahedra).
731 
732   Collective
733 
734   Input Parameters:
735 + comm        - The communicator for the `DM` object
736 . dim         - The spatial dimension of the box, so the resulting mesh is has dimension `dim`-1
737 . faces       - Number of faces per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
738 . lower       - The lower left corner, or `NULL` for (0, 0, 0)
739 . upper       - The upper right corner, or `NULL` for (1, 1, 1)
740 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
741 
742   Output Parameter:
743 . dm - The `DM` object
744 
745   Level: beginner
746 
747 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateBoxMesh()`, `DMPlexCreateFromFile()`, `DMSetType()`, `DMCreate()`
748 @*/
749 PetscErrorCode DMPlexCreateBoxSurfaceMesh(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate, DM *dm)
750 {
751   PetscInt  fac[3] = {1, 1, 1};
752   PetscReal low[3] = {0, 0, 0};
753   PetscReal upp[3] = {1, 1, 1};
754 
755   PetscFunctionBegin;
756   PetscCall(DMCreate(comm, dm));
757   PetscCall(DMSetType(*dm, DMPLEX));
758   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(*dm, dim, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, interpolate));
759   PetscFunctionReturn(PETSC_SUCCESS);
760 }
761 
762 static PetscErrorCode DMPlexCreateLineMesh_Internal(DM dm, PetscInt segments, PetscReal lower, PetscReal upper, DMBoundaryType bd)
763 {
764   PetscInt     i, fStart, fEnd, numCells = 0, numVerts = 0;
765   PetscInt     numPoints[2], *coneSize, *cones, *coneOrientations;
766   PetscScalar *vertexCoords;
767   PetscReal    L, maxCell;
768   PetscBool    markerSeparate = PETSC_FALSE;
769   PetscInt     markerLeft = 1, faceMarkerLeft = 1;
770   PetscInt     markerRight = 1, faceMarkerRight = 2;
771   PetscBool    wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
772   PetscMPIInt  rank;
773 
774   PetscFunctionBegin;
775   PetscAssertPointer(dm, 1);
776 
777   PetscCall(DMSetDimension(dm, 1));
778   PetscCall(DMCreateLabel(dm, "marker"));
779   PetscCall(DMCreateLabel(dm, "Face Sets"));
780 
781   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
782   if (rank == 0) numCells = segments;
783   if (rank == 0) numVerts = segments + (wrap ? 0 : 1);
784 
785   numPoints[0] = numVerts;
786   numPoints[1] = numCells;
787   PetscCall(PetscMalloc4(numCells + numVerts, &coneSize, numCells * 2, &cones, numCells + numVerts, &coneOrientations, numVerts, &vertexCoords));
788   PetscCall(PetscArrayzero(coneOrientations, numCells + numVerts));
789   for (i = 0; i < numCells; ++i) coneSize[i] = 2;
790   for (i = 0; i < numVerts; ++i) coneSize[numCells + i] = 0;
791   for (i = 0; i < numCells; ++i) {
792     cones[2 * i]     = numCells + i % numVerts;
793     cones[2 * i + 1] = numCells + (i + 1) % numVerts;
794   }
795   for (i = 0; i < numVerts; ++i) vertexCoords[i] = lower + (upper - lower) * ((PetscReal)i / (PetscReal)numCells);
796   PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
797   PetscCall(PetscFree4(coneSize, cones, coneOrientations, vertexCoords));
798 
799   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
800   if (markerSeparate) {
801     markerLeft  = faceMarkerLeft;
802     markerRight = faceMarkerRight;
803   }
804   if (!wrap && rank == 0) {
805     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
806     PetscCall(DMSetLabelValue(dm, "marker", fStart, markerLeft));
807     PetscCall(DMSetLabelValue(dm, "marker", fEnd - 1, markerRight));
808     PetscCall(DMSetLabelValue(dm, "Face Sets", fStart, faceMarkerLeft));
809     PetscCall(DMSetLabelValue(dm, "Face Sets", fEnd - 1, faceMarkerRight));
810   }
811   if (wrap) {
812     L       = upper - lower;
813     maxCell = (PetscReal)1.1 * (L / (PetscReal)PetscMax(1, segments));
814     PetscCall(DMSetPeriodicity(dm, &maxCell, &lower, &L));
815   }
816   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
817   PetscFunctionReturn(PETSC_SUCCESS);
818 }
819 
820 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
821 {
822   DM      boundary, vol;
823   DMLabel bdlabel;
824 
825   PetscFunctionBegin;
826   PetscAssertPointer(dm, 1);
827   for (PetscInt i = 0; i < dim; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
828   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &boundary));
829   PetscCall(DMSetType(boundary, DMPLEX));
830   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(boundary, dim, faces, lower, upper, PETSC_FALSE));
831   PetscCall(DMPlexGenerate(boundary, NULL, interpolate, &vol));
832   PetscCall(DMGetLabel(vol, "marker", &bdlabel));
833   if (bdlabel) PetscCall(DMPlexLabelComplete(vol, bdlabel));
834   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_FALSE, vol));
835   PetscCall(DMPlexReplace_Internal(dm, &vol));
836   PetscCall(DMDestroy(&boundary));
837   PetscFunctionReturn(PETSC_SUCCESS);
838 }
839 
840 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
841 {
842   DMLabel     cutLabel  = NULL;
843   PetscInt    markerTop = 1, faceMarkerTop = 1;
844   PetscInt    markerBottom = 1, faceMarkerBottom = 1;
845   PetscInt    markerFront = 1, faceMarkerFront = 1;
846   PetscInt    markerBack = 1, faceMarkerBack = 1;
847   PetscInt    markerRight = 1, faceMarkerRight = 1;
848   PetscInt    markerLeft = 1, faceMarkerLeft = 1;
849   PetscInt    dim;
850   PetscBool   markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
851   PetscMPIInt rank;
852 
853   PetscFunctionBegin;
854   PetscCall(DMGetDimension(dm, &dim));
855   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
856   PetscCall(DMCreateLabel(dm, "marker"));
857   PetscCall(DMCreateLabel(dm, "Face Sets"));
858   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
859   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST || bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST || bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
860     if (cutMarker) {
861       PetscCall(DMCreateLabel(dm, "periodic_cut"));
862       PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
863     }
864   }
865   switch (dim) {
866   case 2:
867     faceMarkerTop    = 3;
868     faceMarkerBottom = 1;
869     faceMarkerRight  = 2;
870     faceMarkerLeft   = 4;
871     break;
872   case 3:
873     faceMarkerBottom = 1;
874     faceMarkerTop    = 2;
875     faceMarkerFront  = 3;
876     faceMarkerBack   = 4;
877     faceMarkerRight  = 5;
878     faceMarkerLeft   = 6;
879     break;
880   default:
881     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
882   }
883   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
884   if (markerSeparate) {
885     markerBottom = faceMarkerBottom;
886     markerTop    = faceMarkerTop;
887     markerFront  = faceMarkerFront;
888     markerBack   = faceMarkerBack;
889     markerRight  = faceMarkerRight;
890     markerLeft   = faceMarkerLeft;
891   }
892   {
893     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
894     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
895     const PetscInt numZEdges    = rank == 0 ? edges[2] : 0;
896     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0] + 1) : 0;
897     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1] + 1) : 0;
898     const PetscInt numZVertices = rank == 0 ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2] + 1) : 0;
899     const PetscInt numCells     = numXEdges * numYEdges * numZEdges;
900     const PetscInt numXFaces    = numYEdges * numZEdges;
901     const PetscInt numYFaces    = numXEdges * numZEdges;
902     const PetscInt numZFaces    = numXEdges * numYEdges;
903     const PetscInt numTotXFaces = numXVertices * numXFaces;
904     const PetscInt numTotYFaces = numYVertices * numYFaces;
905     const PetscInt numTotZFaces = numZVertices * numZFaces;
906     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
907     const PetscInt numTotXEdges = numXEdges * numYVertices * numZVertices;
908     const PetscInt numTotYEdges = numYEdges * numXVertices * numZVertices;
909     const PetscInt numTotZEdges = numZEdges * numXVertices * numYVertices;
910     const PetscInt numVertices  = numXVertices * numYVertices * numZVertices;
911     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
912     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
913     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
914     const PetscInt firstYFace   = firstXFace + numTotXFaces;
915     const PetscInt firstZFace   = firstYFace + numTotYFaces;
916     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
917     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
918     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
919     Vec            coordinates;
920     PetscSection   coordSection;
921     PetscScalar   *coords;
922     PetscInt       coordSize;
923     PetscInt       v, vx, vy, vz;
924     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
925 
926     PetscCall(DMPlexSetChart(dm, 0, numCells + numFaces + numEdges + numVertices));
927     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
928     for (f = firstXFace; f < firstXFace + numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
929     for (e = firstXEdge; e < firstXEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
930     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
931     /* Build cells */
932     for (fz = 0; fz < numZEdges; ++fz) {
933       for (fy = 0; fy < numYEdges; ++fy) {
934         for (fx = 0; fx < numXEdges; ++fx) {
935           PetscInt cell  = (fz * numYEdges + fy) * numXEdges + fx;
936           PetscInt faceB = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
937           PetscInt faceT = firstZFace + (fy * numXEdges + fx) * numZVertices + ((fz + 1) % numZVertices);
938           PetscInt faceF = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
939           PetscInt faceK = firstYFace + (fz * numXEdges + fx) * numYVertices + ((fy + 1) % numYVertices);
940           PetscInt faceL = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
941           PetscInt faceR = firstXFace + (fz * numYEdges + fy) * numXVertices + ((fx + 1) % numXVertices);
942           /* B,  T,  F,  K,  R,  L */
943           PetscInt ornt[6] = {-2, 0, 0, -3, 0, -2}; /* ??? */
944           PetscInt cone[6];
945 
946           /* no boundary twisting in 3D */
947           cone[0] = faceB;
948           cone[1] = faceT;
949           cone[2] = faceF;
950           cone[3] = faceK;
951           cone[4] = faceR;
952           cone[5] = faceL;
953           PetscCall(DMPlexSetCone(dm, cell, cone));
954           PetscCall(DMPlexSetConeOrientation(dm, cell, ornt));
955           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
956           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
957           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
958         }
959       }
960     }
961     /* Build x faces */
962     for (fz = 0; fz < numZEdges; ++fz) {
963       for (fy = 0; fy < numYEdges; ++fy) {
964         for (fx = 0; fx < numXVertices; ++fx) {
965           PetscInt face    = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
966           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
967           PetscInt edgeR   = firstZEdge + (((fy + 1) % numYVertices) * numXVertices + fx) * numZEdges + fz;
968           PetscInt edgeB   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
969           PetscInt edgeT   = firstYEdge + (((fz + 1) % numZVertices) * numXVertices + fx) * numYEdges + fy;
970           PetscInt ornt[4] = {0, 0, -1, -1};
971           PetscInt cone[4];
972 
973           if (dim == 3) {
974             /* markers */
975             if (bdX != DM_BOUNDARY_PERIODIC) {
976               if (fx == numXVertices - 1) {
977                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight));
978                 PetscCall(DMSetLabelValue(dm, "marker", face, markerRight));
979               } else if (fx == 0) {
980                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft));
981                 PetscCall(DMSetLabelValue(dm, "marker", face, markerLeft));
982               }
983             }
984           }
985           cone[0] = edgeB;
986           cone[1] = edgeR;
987           cone[2] = edgeT;
988           cone[3] = edgeL;
989           PetscCall(DMPlexSetCone(dm, face, cone));
990           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
991         }
992       }
993     }
994     /* Build y faces */
995     for (fz = 0; fz < numZEdges; ++fz) {
996       for (fx = 0; fx < numXEdges; ++fx) {
997         for (fy = 0; fy < numYVertices; ++fy) {
998           PetscInt face    = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
999           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
1000           PetscInt edgeR   = firstZEdge + (fy * numXVertices + ((fx + 1) % numXVertices)) * numZEdges + fz;
1001           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1002           PetscInt edgeT   = firstXEdge + (((fz + 1) % numZVertices) * numYVertices + fy) * numXEdges + fx;
1003           PetscInt ornt[4] = {0, 0, -1, -1};
1004           PetscInt cone[4];
1005 
1006           if (dim == 3) {
1007             /* markers */
1008             if (bdY != DM_BOUNDARY_PERIODIC) {
1009               if (fy == numYVertices - 1) {
1010                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack));
1011                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBack));
1012               } else if (fy == 0) {
1013                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront));
1014                 PetscCall(DMSetLabelValue(dm, "marker", face, markerFront));
1015               }
1016             }
1017           }
1018           cone[0] = edgeB;
1019           cone[1] = edgeR;
1020           cone[2] = edgeT;
1021           cone[3] = edgeL;
1022           PetscCall(DMPlexSetCone(dm, face, cone));
1023           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1024         }
1025       }
1026     }
1027     /* Build z faces */
1028     for (fy = 0; fy < numYEdges; ++fy) {
1029       for (fx = 0; fx < numXEdges; ++fx) {
1030         for (fz = 0; fz < numZVertices; fz++) {
1031           PetscInt face    = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
1032           PetscInt edgeL   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
1033           PetscInt edgeR   = firstYEdge + (fz * numXVertices + ((fx + 1) % numXVertices)) * numYEdges + fy;
1034           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1035           PetscInt edgeT   = firstXEdge + (fz * numYVertices + ((fy + 1) % numYVertices)) * numXEdges + fx;
1036           PetscInt ornt[4] = {0, 0, -1, -1};
1037           PetscInt cone[4];
1038 
1039           if (dim == 2) {
1040             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges - 1) {
1041               edgeR += numYEdges - 1 - 2 * fy;
1042               ornt[1] = -1;
1043             }
1044             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges - 1) {
1045               edgeT += numXEdges - 1 - 2 * fx;
1046               ornt[2] = 0;
1047             }
1048             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1049             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1050           } else {
1051             /* markers */
1052             if (bdZ != DM_BOUNDARY_PERIODIC) {
1053               if (fz == numZVertices - 1) {
1054                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop));
1055                 PetscCall(DMSetLabelValue(dm, "marker", face, markerTop));
1056               } else if (fz == 0) {
1057                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom));
1058                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBottom));
1059               }
1060             }
1061           }
1062           cone[0] = edgeB;
1063           cone[1] = edgeR;
1064           cone[2] = edgeT;
1065           cone[3] = edgeL;
1066           PetscCall(DMPlexSetCone(dm, face, cone));
1067           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1068         }
1069       }
1070     }
1071     /* Build Z edges*/
1072     for (vy = 0; vy < numYVertices; vy++) {
1073       for (vx = 0; vx < numXVertices; vx++) {
1074         for (ez = 0; ez < numZEdges; ez++) {
1075           const PetscInt edge    = firstZEdge + (vy * numXVertices + vx) * numZEdges + ez;
1076           const PetscInt vertexB = firstVertex + (ez * numYVertices + vy) * numXVertices + vx;
1077           const PetscInt vertexT = firstVertex + (((ez + 1) % numZVertices) * numYVertices + vy) * numXVertices + vx;
1078           PetscInt       cone[2];
1079 
1080           cone[0] = vertexB;
1081           cone[1] = vertexT;
1082           PetscCall(DMPlexSetCone(dm, edge, cone));
1083           if (dim == 3) {
1084             if (bdX != DM_BOUNDARY_PERIODIC) {
1085               if (vx == numXVertices - 1) {
1086                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1087                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1088                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1089               } else if (vx == 0) {
1090                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1091                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1092                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1093               }
1094             }
1095             if (bdY != DM_BOUNDARY_PERIODIC) {
1096               if (vy == numYVertices - 1) {
1097                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1098                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1099                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1100               } else if (vy == 0) {
1101                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1102                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1103                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1104               }
1105             }
1106           }
1107         }
1108       }
1109     }
1110     /* Build Y edges*/
1111     for (vz = 0; vz < numZVertices; vz++) {
1112       for (vx = 0; vx < numXVertices; vx++) {
1113         for (ey = 0; ey < numYEdges; ey++) {
1114           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges - 1) ? (numXVertices - vx - 1) : (vz * numYVertices + ((ey + 1) % numYVertices)) * numXVertices + vx;
1115           const PetscInt edge    = firstYEdge + (vz * numXVertices + vx) * numYEdges + ey;
1116           const PetscInt vertexF = firstVertex + (vz * numYVertices + ey) * numXVertices + vx;
1117           const PetscInt vertexK = firstVertex + nextv;
1118           PetscInt       cone[2];
1119 
1120           cone[0] = vertexF;
1121           cone[1] = vertexK;
1122           PetscCall(DMPlexSetCone(dm, edge, cone));
1123           if (dim == 2) {
1124             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1125               if (vx == numXVertices - 1) {
1126                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight));
1127                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1128                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1129                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1130               } else if (vx == 0) {
1131                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft));
1132                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1133                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1134                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1135               }
1136             } else {
1137               if (vx == 0 && cutLabel) {
1138                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1139                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1140                 if (ey == numYEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1141               }
1142             }
1143           } else {
1144             if (bdX != DM_BOUNDARY_PERIODIC) {
1145               if (vx == numXVertices - 1) {
1146                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1147                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1148                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1149               } else if (vx == 0) {
1150                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1151                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1152                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1153               }
1154             }
1155             if (bdZ != DM_BOUNDARY_PERIODIC) {
1156               if (vz == numZVertices - 1) {
1157                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1158                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1159                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1160               } else if (vz == 0) {
1161                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1162                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1163                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1164               }
1165             }
1166           }
1167         }
1168       }
1169     }
1170     /* Build X edges*/
1171     for (vz = 0; vz < numZVertices; vz++) {
1172       for (vy = 0; vy < numYVertices; vy++) {
1173         for (ex = 0; ex < numXEdges; ex++) {
1174           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges - 1) ? (numYVertices - vy - 1) * numXVertices : (vz * numYVertices + vy) * numXVertices + (ex + 1) % numXVertices;
1175           const PetscInt edge    = firstXEdge + (vz * numYVertices + vy) * numXEdges + ex;
1176           const PetscInt vertexL = firstVertex + (vz * numYVertices + vy) * numXVertices + ex;
1177           const PetscInt vertexR = firstVertex + nextv;
1178           PetscInt       cone[2];
1179 
1180           cone[0] = vertexL;
1181           cone[1] = vertexR;
1182           PetscCall(DMPlexSetCone(dm, edge, cone));
1183           if (dim == 2) {
1184             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1185               if (vy == numYVertices - 1) {
1186                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop));
1187                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1188                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1189                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1190               } else if (vy == 0) {
1191                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom));
1192                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1193                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1194                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1195               }
1196             } else {
1197               if (vy == 0 && cutLabel) {
1198                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1199                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1200                 if (ex == numXEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1201               }
1202             }
1203           } else {
1204             if (bdY != DM_BOUNDARY_PERIODIC) {
1205               if (vy == numYVertices - 1) {
1206                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1207                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1208                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1209               } else if (vy == 0) {
1210                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1211                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1212                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1213               }
1214             }
1215             if (bdZ != DM_BOUNDARY_PERIODIC) {
1216               if (vz == numZVertices - 1) {
1217                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1218                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1219                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1220               } else if (vz == 0) {
1221                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1222                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1223                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1224               }
1225             }
1226           }
1227         }
1228       }
1229     }
1230     PetscCall(DMPlexSymmetrize(dm));
1231     PetscCall(DMPlexStratify(dm));
1232     /* Build coordinates */
1233     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1234     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1235     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1236     PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVertices));
1237     for (v = firstVertex; v < firstVertex + numVertices; ++v) {
1238       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1239       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1240     }
1241     PetscCall(PetscSectionSetUp(coordSection));
1242     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1243     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1244     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1245     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1246     PetscCall(VecSetBlockSize(coordinates, dim));
1247     PetscCall(VecSetType(coordinates, VECSTANDARD));
1248     PetscCall(VecGetArray(coordinates, &coords));
1249     for (vz = 0; vz < numZVertices; ++vz) {
1250       for (vy = 0; vy < numYVertices; ++vy) {
1251         for (vx = 0; vx < numXVertices; ++vx) {
1252           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * vx;
1253           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * vy;
1254           if (dim == 3) coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 2] = lower[2] + ((upper[2] - lower[2]) / numZEdges) * vz;
1255         }
1256       }
1257     }
1258     PetscCall(VecRestoreArray(coordinates, &coords));
1259     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1260     PetscCall(VecDestroy(&coordinates));
1261   }
1262   PetscFunctionReturn(PETSC_SUCCESS);
1263 }
1264 
1265 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1266 {
1267   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1268   PetscInt       fac[3] = {0, 0, 0}, d;
1269 
1270   PetscFunctionBegin;
1271   PetscAssertPointer(dm, 1);
1272   PetscValidLogicalCollectiveInt(dm, dim, 2);
1273   PetscCall(DMSetDimension(dm, dim));
1274   for (d = 0; d < dim; ++d) {
1275     fac[d] = faces[d];
1276     bdt[d] = periodicity[d];
1277   }
1278   PetscCall(DMPlexCreateCubeMesh_Internal(dm, lower, upper, fac, bdt[0], bdt[1], bdt[2]));
1279   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST || periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST || (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
1280     PetscReal L[3]       = {-1., -1., 0.};
1281     PetscReal maxCell[3] = {-1., -1., 0.};
1282 
1283     for (d = 0; d < dim; ++d) {
1284       if (periodicity[d] != DM_BOUNDARY_NONE) {
1285         L[d]       = upper[d] - lower[d];
1286         maxCell[d] = 1.1 * (L[d] / PetscMax(1, faces[d]));
1287       }
1288     }
1289     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1290   }
1291   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
1292   PetscFunctionReturn(PETSC_SUCCESS);
1293 }
1294 
1295 static PetscErrorCode DMPlexCreateBoxMesh_Internal(DM dm, DMPlexShape shape, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1296 {
1297   PetscFunctionBegin;
1298   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
1299   if (shape == DM_SHAPE_ZBOX) PetscCall(DMPlexCreateBoxMesh_Tensor_SFC_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1300   else if (dim == 1) PetscCall(DMPlexCreateLineMesh_Internal(dm, faces[0], lower[0], upper[0], periodicity[0]));
1301   else if (simplex) PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1302   else PetscCall(DMPlexCreateBoxMesh_Tensor_Internal(dm, dim, faces, lower, upper, periodicity));
1303   if (!interpolate && dim > 1 && !simplex) {
1304     DM udm;
1305 
1306     PetscCall(DMPlexUninterpolate(dm, &udm));
1307     PetscCall(DMPlexCopyCoordinates(dm, udm));
1308     PetscCall(DMPlexReplace_Internal(dm, &udm));
1309   }
1310   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
1311   PetscFunctionReturn(PETSC_SUCCESS);
1312 }
1313 
1314 /*@C
1315   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
1316 
1317   Collective
1318 
1319   Input Parameters:
1320 + comm        - The communicator for the `DM` object
1321 . dim         - The spatial dimension
1322 . simplex     - `PETSC_TRUE` for simplices, `PETSC_FALSE` for tensor cells
1323 . faces       - Number of faces per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1324 . lower       - The lower left corner, or `NULL` for (0, 0, 0)
1325 . upper       - The upper right corner, or `NULL` for (1, 1, 1)
1326 . periodicity - The boundary type for the X,Y,Z direction, or `NULL` for `DM_BOUNDARY_NONE`
1327 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1328 
1329   Output Parameter:
1330 . dm - The `DM` object
1331 
1332   Level: beginner
1333 
1334   Note:
1335   To customize this mesh using options, use
1336 .vb
1337   DMCreate(comm, &dm);
1338   DMSetType(dm, DMPLEX);
1339   DMSetFromOptions(dm);
1340 .ve
1341   and use the options in `DMSetFromOptions()`.
1342 
1343   Here is the numbering returned for 2 faces in each direction for tensor cells\:
1344 .vb
1345  10---17---11---18----12
1346   |         |         |
1347   |         |         |
1348  20    2   22    3    24
1349   |         |         |
1350   |         |         |
1351   7---15----8---16----9
1352   |         |         |
1353   |         |         |
1354  19    0   21    1   23
1355   |         |         |
1356   |         |         |
1357   4---13----5---14----6
1358 .ve
1359   and for simplicial cells
1360 .vb
1361  14----8---15----9----16
1362   |\     5  |\      7 |
1363   | \       | \       |
1364  13   2    14    3    15
1365   | 4   \   | 6   \   |
1366   |       \ |       \ |
1367  11----6---12----7----13
1368   |\        |\        |
1369   | \    1  | \     3 |
1370  10   0    11    1    12
1371   | 0   \   | 2   \   |
1372   |       \ |       \ |
1373   8----4----9----5----10
1374 .ve
1375 
1376 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
1377 @*/
1378 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
1379 {
1380   PetscInt       fac[3] = {1, 1, 1};
1381   PetscReal      low[3] = {0, 0, 0};
1382   PetscReal      upp[3] = {1, 1, 1};
1383   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1384 
1385   PetscFunctionBegin;
1386   PetscCall(DMCreate(comm, dm));
1387   PetscCall(DMSetType(*dm, DMPLEX));
1388   PetscCall(DMPlexCreateBoxMesh_Internal(*dm, DM_SHAPE_BOX, dim, simplex, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt, interpolate));
1389   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1390   PetscFunctionReturn(PETSC_SUCCESS);
1391 }
1392 
1393 static PetscErrorCode DMPlexCreateWedgeBoxMesh_Internal(DM dm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1394 {
1395   DM       bdm, vol;
1396   PetscInt i;
1397 
1398   PetscFunctionBegin;
1399   // TODO Now we can support periodicity
1400   for (i = 0; i < 3; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity not yet supported");
1401   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &bdm));
1402   PetscCall(DMSetType(bdm, DMPLEX));
1403   PetscCall(DMSetDimension(bdm, 2));
1404   PetscCall(PetscLogEventBegin(DMPLEX_Generate, bdm, 0, 0, 0));
1405   PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(bdm, 2, faces, lower, upper, periodicity, PETSC_TRUE));
1406   PetscCall(DMPlexExtrude(bdm, faces[2], upper[2] - lower[2], PETSC_TRUE, PETSC_FALSE, PETSC_FALSE, NULL, NULL, &vol));
1407   PetscCall(PetscLogEventEnd(DMPLEX_Generate, bdm, 0, 0, 0));
1408   PetscCall(DMDestroy(&bdm));
1409   PetscCall(DMPlexReplace_Internal(dm, &vol));
1410   if (lower[2] != 0.0) {
1411     Vec          v;
1412     PetscScalar *x;
1413     PetscInt     cDim, n;
1414 
1415     PetscCall(DMGetCoordinatesLocal(dm, &v));
1416     PetscCall(VecGetBlockSize(v, &cDim));
1417     PetscCall(VecGetLocalSize(v, &n));
1418     PetscCall(VecGetArray(v, &x));
1419     x += cDim;
1420     for (i = 0; i < n; i += cDim) x[i] += lower[2];
1421     PetscCall(VecRestoreArray(v, &x));
1422     PetscCall(DMSetCoordinatesLocal(dm, v));
1423   }
1424   PetscFunctionReturn(PETSC_SUCCESS);
1425 }
1426 
1427 /*@
1428   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tesselating the (x,y) plane and extruding in the third direction using wedge cells.
1429 
1430   Collective
1431 
1432   Input Parameters:
1433 + comm        - The communicator for the `DM` object
1434 . faces       - Number of faces per dimension, or `NULL` for (1, 1, 1)
1435 . lower       - The lower left corner, or `NULL` for (0, 0, 0)
1436 . upper       - The upper right corner, or `NULL` for (1, 1, 1)
1437 . periodicity - The boundary type for the X,Y,Z direction, or `NULL` for `DM_BOUNDARY_NONE`
1438 . orderHeight - If `PETSC_TRUE`, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1439 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1440 
1441   Output Parameter:
1442 . dm - The `DM` object
1443 
1444   Level: beginner
1445 
1446 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateWedgeCylinderMesh()`, `DMExtrude()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1447 @*/
1448 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool orderHeight, PetscBool interpolate, DM *dm)
1449 {
1450   PetscInt       fac[3] = {1, 1, 1};
1451   PetscReal      low[3] = {0, 0, 0};
1452   PetscReal      upp[3] = {1, 1, 1};
1453   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1454 
1455   PetscFunctionBegin;
1456   PetscCall(DMCreate(comm, dm));
1457   PetscCall(DMSetType(*dm, DMPLEX));
1458   PetscCall(DMPlexCreateWedgeBoxMesh_Internal(*dm, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt));
1459   if (!interpolate) {
1460     DM udm;
1461 
1462     PetscCall(DMPlexUninterpolate(*dm, &udm));
1463     PetscCall(DMPlexReplace_Internal(*dm, &udm));
1464   }
1465   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1466   PetscFunctionReturn(PETSC_SUCCESS);
1467 }
1468 
1469 /*
1470   DMPlexTensorPointLexicographic_Private - Returns all tuples of size 'len' with nonnegative integers that are all less than or equal to 'max' for that dimension.
1471 
1472   Input Parameters:
1473 + len - The length of the tuple
1474 . max - The maximum for each dimension, so values are in [0, max)
1475 - tup - A tuple of length len+1: tup[len] > 0 indicates a stopping condition
1476 
1477   Output Parameter:
1478 . tup - A tuple of `len` integers whose entries are at most `max`
1479 
1480   Level: developer
1481 
1482   Note:
1483   Ordering is lexicographic with lowest index as least significant in ordering.
1484   e.g. for len == 2 and max == 2, this will return, in order, {0,0}, {1,0}, {2,0}, {0,1}, {1,1}, {2,1}, {0,2}, {1,2}, {2,2}.
1485 
1486 .seealso: PetscDualSpaceTensorPointLexicographic_Internal(), PetscDualSpaceLatticePointLexicographic_Internal()
1487 */
1488 static PetscErrorCode DMPlexTensorPointLexicographic_Private(PetscInt len, const PetscInt max[], PetscInt tup[])
1489 {
1490   PetscInt i;
1491 
1492   PetscFunctionBegin;
1493   for (i = 0; i < len; ++i) {
1494     if (tup[i] < max[i] - 1) {
1495       break;
1496     } else {
1497       tup[i] = 0;
1498     }
1499   }
1500   if (i == len) tup[i - 1] = max[i - 1];
1501   else ++tup[i];
1502   PetscFunctionReturn(PETSC_SUCCESS);
1503 }
1504 
1505 static PetscInt TupleToIndex_Private(PetscInt len, const PetscInt max[], const PetscInt tup[])
1506 {
1507   PetscInt i, idx = tup[len - 1];
1508 
1509   for (i = len - 2; i >= 0; --i) {
1510     idx *= max[i];
1511     idx += tup[i];
1512   }
1513   return idx;
1514 }
1515 
1516 static PetscErrorCode DestroyExtent_Private(void *extent)
1517 {
1518   return PetscFree(extent);
1519 }
1520 
1521 static PetscErrorCode DMPlexCreateHypercubicMesh_Internal(DM dm, PetscInt dim, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], const DMBoundaryType bd[])
1522 {
1523   Vec          coordinates;
1524   PetscSection coordSection;
1525   DMLabel      cutLabel    = NULL;
1526   PetscBool    cutMarker   = PETSC_FALSE;
1527   PetscBool    periodic    = PETSC_FALSE;
1528   PetscInt     numCells    = 1, c;
1529   PetscInt     numVertices = 1, v;
1530   PetscScalar *coords;
1531   PetscInt    *vertices, *vert, *vtmp, *supp, cone[2];
1532   PetscInt     d, e, cell = 0, coordSize;
1533   PetscMPIInt  rank;
1534 
1535   PetscFunctionBegin;
1536   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1537   PetscCall(DMSetDimension(dm, dim));
1538   PetscCall(PetscCalloc4(dim, &vertices, dim, &vert, dim, &vtmp, 2 * dim, &supp));
1539   PetscCall(DMCreateLabel(dm, "marker"));
1540   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
1541   for (d = 0; d < dim; ++d) periodic = (periodic || bd[d] == DM_BOUNDARY_PERIODIC) ? PETSC_TRUE : PETSC_FALSE;
1542   if (periodic && cutMarker) {
1543     PetscCall(DMCreateLabel(dm, "periodic_cut"));
1544     PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
1545   }
1546   for (d = 0; d < dim; ++d) PetscCheck(bd[d] == DM_BOUNDARY_PERIODIC, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Hypercubic mesh must be periodic now");
1547   for (d = 0; d < dim; ++d) {
1548     vertices[d] = edges[d];
1549     numVertices *= vertices[d];
1550   }
1551   numCells = numVertices * dim;
1552   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1553   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, 2));
1554   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetSupportSize(dm, v, 2 * dim));
1555   /* TODO Loop over boundary and reset support sizes */
1556   PetscCall(DMSetUp(dm)); /* Allocate space for cones and supports */
1557   /* Build cell cones and vertex supports */
1558   PetscCall(DMCreateLabel(dm, "celltype"));
1559   while (vert[dim - 1] < vertices[dim - 1]) {
1560     const PetscInt vertex = TupleToIndex_Private(dim, vertices, vert) + numCells;
1561     PetscInt       s      = 0;
1562 
1563     PetscCall(PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT ":", vertex));
1564     for (d = 0; d < dim; ++d) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, vert[d]));
1565     PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
1566     PetscCall(DMPlexSetCellType(dm, vertex, DM_POLYTOPE_POINT));
1567     for (d = 0; d < dim; ++d) {
1568       for (e = 0; e < dim; ++e) vtmp[e] = vert[e];
1569       vtmp[d] = (vert[d] + 1) % vertices[d];
1570       cone[0] = vertex;
1571       cone[1] = TupleToIndex_Private(dim, vertices, vtmp) + numCells;
1572       PetscCall(PetscPrintf(PETSC_COMM_SELF, "  Vertex %" PetscInt_FMT ":", cone[1]));
1573       for (e = 0; e < dim; ++e) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, vtmp[e]));
1574       PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
1575       PetscCall(DMPlexSetCone(dm, cell, cone));
1576       PetscCall(DMPlexSetCellType(dm, cell, DM_POLYTOPE_SEGMENT));
1577       PetscCall(PetscPrintf(PETSC_COMM_SELF, "  Edge %" PetscInt_FMT " (%" PetscInt_FMT " %" PetscInt_FMT ")\n", cell, cone[0], cone[1]));
1578       ++cell;
1579     }
1580     for (d = 0; d < dim; ++d) {
1581       for (e = 0; e < dim; ++e) vtmp[e] = vert[e];
1582       vtmp[d]   = (vert[d] + vertices[d] - 1) % vertices[d];
1583       supp[s++] = TupleToIndex_Private(dim, vertices, vtmp) * dim + d;
1584       supp[s++] = (vertex - numCells) * dim + d;
1585       PetscCall(DMPlexSetSupport(dm, vertex, supp));
1586     }
1587     PetscCall(DMPlexTensorPointLexicographic_Private(dim, vertices, vert));
1588   }
1589   PetscCall(DMPlexStratify(dm));
1590   /* Build coordinates */
1591   PetscCall(DMGetCoordinateSection(dm, &coordSection));
1592   PetscCall(PetscSectionSetNumFields(coordSection, 1));
1593   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1594   PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1595   for (v = numCells; v < numCells + numVertices; ++v) {
1596     PetscCall(PetscSectionSetDof(coordSection, v, dim));
1597     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1598   }
1599   PetscCall(PetscSectionSetUp(coordSection));
1600   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1601   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1602   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1603   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1604   PetscCall(VecSetBlockSize(coordinates, dim));
1605   PetscCall(VecSetType(coordinates, VECSTANDARD));
1606   PetscCall(VecGetArray(coordinates, &coords));
1607   for (d = 0; d < dim; ++d) vert[d] = 0;
1608   while (vert[dim - 1] < vertices[dim - 1]) {
1609     const PetscInt vertex = TupleToIndex_Private(dim, vertices, vert);
1610 
1611     for (d = 0; d < dim; ++d) coords[vertex * dim + d] = lower[d] + ((upper[d] - lower[d]) / vertices[d]) * vert[d];
1612     PetscCall(PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT ":", vertex));
1613     for (d = 0; d < dim; ++d) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, vert[d]));
1614     for (d = 0; d < dim; ++d) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %g", (double)PetscRealPart(coords[vertex * dim + d])));
1615     PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
1616     PetscCall(DMPlexTensorPointLexicographic_Private(dim, vertices, vert));
1617   }
1618   PetscCall(VecRestoreArray(coordinates, &coords));
1619   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1620   PetscCall(VecDestroy(&coordinates));
1621   PetscCall(PetscFree4(vertices, vert, vtmp, supp));
1622   //PetscCall(DMSetPeriodicity(dm, NULL, lower, upper));
1623   // Attach the extent
1624   {
1625     PetscContainer c;
1626     PetscInt      *extent;
1627 
1628     PetscCall(PetscMalloc1(dim, &extent));
1629     for (PetscInt d = 0; d < dim; ++d) extent[d] = edges[d];
1630     PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &c));
1631     PetscCall(PetscContainerSetUserDestroy(c, DestroyExtent_Private));
1632     PetscCall(PetscContainerSetPointer(c, extent));
1633     PetscCall(PetscObjectCompose((PetscObject)dm, "_extent", (PetscObject)c));
1634     PetscCall(PetscContainerDestroy(&c));
1635   }
1636   PetscFunctionReturn(PETSC_SUCCESS);
1637 }
1638 
1639 /*@C
1640   DMPlexCreateHypercubicMesh - Creates a periodic mesh on the tensor product of unit intervals using only vertices and edges.
1641 
1642   Collective
1643 
1644   Input Parameters:
1645 + comm  - The communicator for the DM object
1646 . dim   - The spatial dimension
1647 . edges - Number of edges per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1648 . lower - The lower left corner, or `NULL` for (0, 0, 0)
1649 - upper - The upper right corner, or `NULL` for (1, 1, 1)
1650 
1651   Output Parameter:
1652 . dm - The DM object
1653 
1654   Level: beginner
1655 
1656   Note:
1657   If you want to customize this mesh using options, you just need to
1658 .vb
1659   DMCreate(comm, &dm);
1660   DMSetType(dm, DMPLEX);
1661   DMSetFromOptions(dm);
1662 .ve
1663   and use the options on the `DMSetFromOptions()` page.
1664 
1665   The vertices are numbered is lexicographic order, and the dim edges exiting a vertex in the positive orthant are number consecutively,
1666 .vb
1667  18--0-19--2-20--4-18
1668   |     |     |     |
1669  13    15    17    13
1670   |     |     |     |
1671  24-12-25-14-26-16-24
1672   |     |     |     |
1673   7     9    11     7
1674   |     |     |     |
1675  21--6-22--8-23-10-21
1676   |     |     |     |
1677   1     3     5     1
1678   |     |     |     |
1679  18--0-19--2-20--4-18
1680 .ve
1681 
1682 .seealso: `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
1683 @*/
1684 PetscErrorCode DMPlexCreateHypercubicMesh(MPI_Comm comm, PetscInt dim, const PetscInt edges[], const PetscReal lower[], const PetscReal upper[], DM *dm)
1685 {
1686   PetscInt       *edg;
1687   PetscReal      *low, *upp;
1688   DMBoundaryType *bdt;
1689   PetscInt        d;
1690 
1691   PetscFunctionBegin;
1692   PetscCall(DMCreate(comm, dm));
1693   PetscCall(DMSetType(*dm, DMPLEX));
1694   PetscCall(PetscMalloc4(dim, &edg, dim, &low, dim, &upp, dim, &bdt));
1695   for (d = 0; d < dim; ++d) {
1696     edg[d] = edges ? edges[d] : 1;
1697     low[d] = lower ? lower[d] : 0.;
1698     upp[d] = upper ? upper[d] : 1.;
1699     bdt[d] = DM_BOUNDARY_PERIODIC;
1700   }
1701   PetscCall(DMPlexCreateHypercubicMesh_Internal(*dm, dim, low, upp, edg, bdt));
1702   PetscCall(PetscFree4(edg, low, upp, bdt));
1703   PetscFunctionReturn(PETSC_SUCCESS);
1704 }
1705 
1706 /*@C
1707   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all `DM` options in the database.
1708 
1709   Logically Collective
1710 
1711   Input Parameters:
1712 + dm     - the `DM` context
1713 - prefix - the prefix to prepend to all option names
1714 
1715   Level: advanced
1716 
1717   Note:
1718   A hyphen (-) must NOT be given at the beginning of the prefix name.
1719   The first character of all runtime options is AUTOMATICALLY the hyphen.
1720 
1721 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `SNESSetFromOptions()`
1722 @*/
1723 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
1724 {
1725   DM_Plex *mesh = (DM_Plex *)dm->data;
1726 
1727   PetscFunctionBegin;
1728   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1729   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, prefix));
1730   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)mesh->partitioner, prefix));
1731   PetscFunctionReturn(PETSC_SUCCESS);
1732 }
1733 
1734 /* Remap geometry to cylinder
1735    TODO: This only works for a single refinement, then it is broken
1736 
1737      Interior square: Linear interpolation is correct
1738      The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
1739      such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
1740 
1741        phi     = arctan(y/x)
1742        d_close = sqrt(1/8 + 1/4 sin^2(phi))
1743        d_far   = sqrt(1/2 + sin^2(phi))
1744 
1745      so we remap them using
1746 
1747        x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
1748        y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
1749 
1750      If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
1751 */
1752 static void snapToCylinder(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
1753 {
1754   const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1755   const PetscReal ds2 = 0.5 * dis;
1756 
1757   if ((PetscAbsScalar(u[0]) <= ds2) && (PetscAbsScalar(u[1]) <= ds2)) {
1758     f0[0] = u[0];
1759     f0[1] = u[1];
1760   } else {
1761     PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
1762 
1763     x    = PetscRealPart(u[0]);
1764     y    = PetscRealPart(u[1]);
1765     phi  = PetscAtan2Real(y, x);
1766     sinp = PetscSinReal(phi);
1767     cosp = PetscCosReal(phi);
1768     if ((PetscAbsReal(phi) > PETSC_PI / 4.0) && (PetscAbsReal(phi) < 3.0 * PETSC_PI / 4.0)) {
1769       dc = PetscAbsReal(ds2 / sinp);
1770       df = PetscAbsReal(dis / sinp);
1771       xc = ds2 * x / PetscAbsReal(y);
1772       yc = ds2 * PetscSignReal(y);
1773     } else {
1774       dc = PetscAbsReal(ds2 / cosp);
1775       df = PetscAbsReal(dis / cosp);
1776       xc = ds2 * PetscSignReal(x);
1777       yc = ds2 * y / PetscAbsReal(x);
1778     }
1779     f0[0] = xc + (u[0] - xc) * (1.0 - dc) / (df - dc);
1780     f0[1] = yc + (u[1] - yc) * (1.0 - dc) / (df - dc);
1781   }
1782   f0[2] = u[2];
1783 }
1784 
1785 static PetscErrorCode DMPlexCreateHexCylinderMesh_Internal(DM dm, DMBoundaryType periodicZ)
1786 {
1787   const PetscInt dim = 3;
1788   PetscInt       numCells, numVertices;
1789   PetscMPIInt    rank;
1790 
1791   PetscFunctionBegin;
1792   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
1793   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1794   PetscCall(DMSetDimension(dm, dim));
1795   /* Create topology */
1796   {
1797     PetscInt cone[8], c;
1798 
1799     numCells    = rank == 0 ? 5 : 0;
1800     numVertices = rank == 0 ? 16 : 0;
1801     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1802       numCells *= 3;
1803       numVertices = rank == 0 ? 24 : 0;
1804     }
1805     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1806     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 8));
1807     PetscCall(DMSetUp(dm));
1808     if (rank == 0) {
1809       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1810         cone[0] = 15;
1811         cone[1] = 18;
1812         cone[2] = 17;
1813         cone[3] = 16;
1814         cone[4] = 31;
1815         cone[5] = 32;
1816         cone[6] = 33;
1817         cone[7] = 34;
1818         PetscCall(DMPlexSetCone(dm, 0, cone));
1819         cone[0] = 16;
1820         cone[1] = 17;
1821         cone[2] = 24;
1822         cone[3] = 23;
1823         cone[4] = 32;
1824         cone[5] = 36;
1825         cone[6] = 37;
1826         cone[7] = 33; /* 22 25 26 21 */
1827         PetscCall(DMPlexSetCone(dm, 1, cone));
1828         cone[0] = 18;
1829         cone[1] = 27;
1830         cone[2] = 24;
1831         cone[3] = 17;
1832         cone[4] = 34;
1833         cone[5] = 33;
1834         cone[6] = 37;
1835         cone[7] = 38;
1836         PetscCall(DMPlexSetCone(dm, 2, cone));
1837         cone[0] = 29;
1838         cone[1] = 27;
1839         cone[2] = 18;
1840         cone[3] = 15;
1841         cone[4] = 35;
1842         cone[5] = 31;
1843         cone[6] = 34;
1844         cone[7] = 38;
1845         PetscCall(DMPlexSetCone(dm, 3, cone));
1846         cone[0] = 29;
1847         cone[1] = 15;
1848         cone[2] = 16;
1849         cone[3] = 23;
1850         cone[4] = 35;
1851         cone[5] = 36;
1852         cone[6] = 32;
1853         cone[7] = 31;
1854         PetscCall(DMPlexSetCone(dm, 4, cone));
1855 
1856         cone[0] = 31;
1857         cone[1] = 34;
1858         cone[2] = 33;
1859         cone[3] = 32;
1860         cone[4] = 19;
1861         cone[5] = 22;
1862         cone[6] = 21;
1863         cone[7] = 20;
1864         PetscCall(DMPlexSetCone(dm, 5, cone));
1865         cone[0] = 32;
1866         cone[1] = 33;
1867         cone[2] = 37;
1868         cone[3] = 36;
1869         cone[4] = 22;
1870         cone[5] = 25;
1871         cone[6] = 26;
1872         cone[7] = 21;
1873         PetscCall(DMPlexSetCone(dm, 6, cone));
1874         cone[0] = 34;
1875         cone[1] = 38;
1876         cone[2] = 37;
1877         cone[3] = 33;
1878         cone[4] = 20;
1879         cone[5] = 21;
1880         cone[6] = 26;
1881         cone[7] = 28;
1882         PetscCall(DMPlexSetCone(dm, 7, cone));
1883         cone[0] = 35;
1884         cone[1] = 38;
1885         cone[2] = 34;
1886         cone[3] = 31;
1887         cone[4] = 30;
1888         cone[5] = 19;
1889         cone[6] = 20;
1890         cone[7] = 28;
1891         PetscCall(DMPlexSetCone(dm, 8, cone));
1892         cone[0] = 35;
1893         cone[1] = 31;
1894         cone[2] = 32;
1895         cone[3] = 36;
1896         cone[4] = 30;
1897         cone[5] = 25;
1898         cone[6] = 22;
1899         cone[7] = 19;
1900         PetscCall(DMPlexSetCone(dm, 9, cone));
1901 
1902         cone[0] = 19;
1903         cone[1] = 20;
1904         cone[2] = 21;
1905         cone[3] = 22;
1906         cone[4] = 15;
1907         cone[5] = 16;
1908         cone[6] = 17;
1909         cone[7] = 18;
1910         PetscCall(DMPlexSetCone(dm, 10, cone));
1911         cone[0] = 22;
1912         cone[1] = 21;
1913         cone[2] = 26;
1914         cone[3] = 25;
1915         cone[4] = 16;
1916         cone[5] = 23;
1917         cone[6] = 24;
1918         cone[7] = 17;
1919         PetscCall(DMPlexSetCone(dm, 11, cone));
1920         cone[0] = 20;
1921         cone[1] = 28;
1922         cone[2] = 26;
1923         cone[3] = 21;
1924         cone[4] = 18;
1925         cone[5] = 17;
1926         cone[6] = 24;
1927         cone[7] = 27;
1928         PetscCall(DMPlexSetCone(dm, 12, cone));
1929         cone[0] = 30;
1930         cone[1] = 28;
1931         cone[2] = 20;
1932         cone[3] = 19;
1933         cone[4] = 29;
1934         cone[5] = 15;
1935         cone[6] = 18;
1936         cone[7] = 27;
1937         PetscCall(DMPlexSetCone(dm, 13, cone));
1938         cone[0] = 30;
1939         cone[1] = 19;
1940         cone[2] = 22;
1941         cone[3] = 25;
1942         cone[4] = 29;
1943         cone[5] = 23;
1944         cone[6] = 16;
1945         cone[7] = 15;
1946         PetscCall(DMPlexSetCone(dm, 14, cone));
1947       } else {
1948         cone[0] = 5;
1949         cone[1] = 8;
1950         cone[2] = 7;
1951         cone[3] = 6;
1952         cone[4] = 9;
1953         cone[5] = 12;
1954         cone[6] = 11;
1955         cone[7] = 10;
1956         PetscCall(DMPlexSetCone(dm, 0, cone));
1957         cone[0] = 6;
1958         cone[1] = 7;
1959         cone[2] = 14;
1960         cone[3] = 13;
1961         cone[4] = 12;
1962         cone[5] = 15;
1963         cone[6] = 16;
1964         cone[7] = 11;
1965         PetscCall(DMPlexSetCone(dm, 1, cone));
1966         cone[0] = 8;
1967         cone[1] = 17;
1968         cone[2] = 14;
1969         cone[3] = 7;
1970         cone[4] = 10;
1971         cone[5] = 11;
1972         cone[6] = 16;
1973         cone[7] = 18;
1974         PetscCall(DMPlexSetCone(dm, 2, cone));
1975         cone[0] = 19;
1976         cone[1] = 17;
1977         cone[2] = 8;
1978         cone[3] = 5;
1979         cone[4] = 20;
1980         cone[5] = 9;
1981         cone[6] = 10;
1982         cone[7] = 18;
1983         PetscCall(DMPlexSetCone(dm, 3, cone));
1984         cone[0] = 19;
1985         cone[1] = 5;
1986         cone[2] = 6;
1987         cone[3] = 13;
1988         cone[4] = 20;
1989         cone[5] = 15;
1990         cone[6] = 12;
1991         cone[7] = 9;
1992         PetscCall(DMPlexSetCone(dm, 4, cone));
1993       }
1994     }
1995     PetscCall(DMPlexSymmetrize(dm));
1996     PetscCall(DMPlexStratify(dm));
1997   }
1998   /* Create cube geometry */
1999   {
2000     Vec             coordinates;
2001     PetscSection    coordSection;
2002     PetscScalar    *coords;
2003     PetscInt        coordSize, v;
2004     const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
2005     const PetscReal ds2 = dis / 2.0;
2006 
2007     /* Build coordinates */
2008     PetscCall(DMGetCoordinateSection(dm, &coordSection));
2009     PetscCall(PetscSectionSetNumFields(coordSection, 1));
2010     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
2011     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
2012     for (v = numCells; v < numCells + numVertices; ++v) {
2013       PetscCall(PetscSectionSetDof(coordSection, v, dim));
2014       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
2015     }
2016     PetscCall(PetscSectionSetUp(coordSection));
2017     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2018     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2019     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2020     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2021     PetscCall(VecSetBlockSize(coordinates, dim));
2022     PetscCall(VecSetType(coordinates, VECSTANDARD));
2023     PetscCall(VecGetArray(coordinates, &coords));
2024     if (rank == 0) {
2025       coords[0 * dim + 0]  = -ds2;
2026       coords[0 * dim + 1]  = -ds2;
2027       coords[0 * dim + 2]  = 0.0;
2028       coords[1 * dim + 0]  = ds2;
2029       coords[1 * dim + 1]  = -ds2;
2030       coords[1 * dim + 2]  = 0.0;
2031       coords[2 * dim + 0]  = ds2;
2032       coords[2 * dim + 1]  = ds2;
2033       coords[2 * dim + 2]  = 0.0;
2034       coords[3 * dim + 0]  = -ds2;
2035       coords[3 * dim + 1]  = ds2;
2036       coords[3 * dim + 2]  = 0.0;
2037       coords[4 * dim + 0]  = -ds2;
2038       coords[4 * dim + 1]  = -ds2;
2039       coords[4 * dim + 2]  = 1.0;
2040       coords[5 * dim + 0]  = -ds2;
2041       coords[5 * dim + 1]  = ds2;
2042       coords[5 * dim + 2]  = 1.0;
2043       coords[6 * dim + 0]  = ds2;
2044       coords[6 * dim + 1]  = ds2;
2045       coords[6 * dim + 2]  = 1.0;
2046       coords[7 * dim + 0]  = ds2;
2047       coords[7 * dim + 1]  = -ds2;
2048       coords[7 * dim + 2]  = 1.0;
2049       coords[8 * dim + 0]  = dis;
2050       coords[8 * dim + 1]  = -dis;
2051       coords[8 * dim + 2]  = 0.0;
2052       coords[9 * dim + 0]  = dis;
2053       coords[9 * dim + 1]  = dis;
2054       coords[9 * dim + 2]  = 0.0;
2055       coords[10 * dim + 0] = dis;
2056       coords[10 * dim + 1] = -dis;
2057       coords[10 * dim + 2] = 1.0;
2058       coords[11 * dim + 0] = dis;
2059       coords[11 * dim + 1] = dis;
2060       coords[11 * dim + 2] = 1.0;
2061       coords[12 * dim + 0] = -dis;
2062       coords[12 * dim + 1] = dis;
2063       coords[12 * dim + 2] = 0.0;
2064       coords[13 * dim + 0] = -dis;
2065       coords[13 * dim + 1] = dis;
2066       coords[13 * dim + 2] = 1.0;
2067       coords[14 * dim + 0] = -dis;
2068       coords[14 * dim + 1] = -dis;
2069       coords[14 * dim + 2] = 0.0;
2070       coords[15 * dim + 0] = -dis;
2071       coords[15 * dim + 1] = -dis;
2072       coords[15 * dim + 2] = 1.0;
2073       if (periodicZ == DM_BOUNDARY_PERIODIC) {
2074         /* 15 31 19 */ coords[16 * dim + 0] = -ds2;
2075         coords[16 * dim + 1]                = -ds2;
2076         coords[16 * dim + 2]                = 0.5;
2077         /* 16 32 22 */ coords[17 * dim + 0] = ds2;
2078         coords[17 * dim + 1]                = -ds2;
2079         coords[17 * dim + 2]                = 0.5;
2080         /* 17 33 21 */ coords[18 * dim + 0] = ds2;
2081         coords[18 * dim + 1]                = ds2;
2082         coords[18 * dim + 2]                = 0.5;
2083         /* 18 34 20 */ coords[19 * dim + 0] = -ds2;
2084         coords[19 * dim + 1]                = ds2;
2085         coords[19 * dim + 2]                = 0.5;
2086         /* 29 35 30 */ coords[20 * dim + 0] = -dis;
2087         coords[20 * dim + 1]                = -dis;
2088         coords[20 * dim + 2]                = 0.5;
2089         /* 23 36 25 */ coords[21 * dim + 0] = dis;
2090         coords[21 * dim + 1]                = -dis;
2091         coords[21 * dim + 2]                = 0.5;
2092         /* 24 37 26 */ coords[22 * dim + 0] = dis;
2093         coords[22 * dim + 1]                = dis;
2094         coords[22 * dim + 2]                = 0.5;
2095         /* 27 38 28 */ coords[23 * dim + 0] = -dis;
2096         coords[23 * dim + 1]                = dis;
2097         coords[23 * dim + 2]                = 0.5;
2098       }
2099     }
2100     PetscCall(VecRestoreArray(coordinates, &coords));
2101     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2102     PetscCall(VecDestroy(&coordinates));
2103   }
2104   /* Create periodicity */
2105   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
2106     PetscReal L[3]       = {-1., -1., 0.};
2107     PetscReal maxCell[3] = {-1., -1., 0.};
2108     PetscReal lower[3]   = {0.0, 0.0, 0.0};
2109     PetscReal upper[3]   = {1.0, 1.0, 1.5};
2110     PetscInt  numZCells  = 3;
2111 
2112     L[2]       = upper[2] - lower[2];
2113     maxCell[2] = 1.1 * (L[2] / numZCells);
2114     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
2115   }
2116   {
2117     DM          cdm;
2118     PetscDS     cds;
2119     PetscScalar c[2] = {1.0, 1.0};
2120 
2121     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_TRUE, snapToCylinder));
2122     PetscCall(DMGetCoordinateDM(dm, &cdm));
2123     PetscCall(DMGetDS(cdm, &cds));
2124     PetscCall(PetscDSSetConstants(cds, 2, c));
2125   }
2126   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
2127 
2128   /* Wait for coordinate creation before doing in-place modification */
2129   PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2130   PetscFunctionReturn(PETSC_SUCCESS);
2131 }
2132 
2133 /*@
2134   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
2135 
2136   Collective
2137 
2138   Input Parameters:
2139 + comm      - The communicator for the `DM` object
2140 - periodicZ - The boundary type for the Z direction
2141 
2142   Output Parameter:
2143 . dm - The `DM` object
2144 
2145   Level: beginner
2146 
2147   Note:
2148   Here is the output numbering looking from the bottom of the cylinder\:
2149 .vb
2150        17-----14
2151         |     |
2152         |  2  |
2153         |     |
2154  17-----8-----7-----14
2155   |     |     |     |
2156   |  3  |  0  |  1  |
2157   |     |     |     |
2158  19-----5-----6-----13
2159         |     |
2160         |  4  |
2161         |     |
2162        19-----13
2163 
2164  and up through the top
2165 
2166        18-----16
2167         |     |
2168         |  2  |
2169         |     |
2170  18----10----11-----16
2171   |     |     |     |
2172   |  3  |  0  |  1  |
2173   |     |     |     |
2174  20-----9----12-----15
2175         |     |
2176         |  4  |
2177         |     |
2178        20-----15
2179 .ve
2180 
2181 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2182 @*/
2183 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, DMBoundaryType periodicZ, DM *dm)
2184 {
2185   PetscFunctionBegin;
2186   PetscAssertPointer(dm, 3);
2187   PetscCall(DMCreate(comm, dm));
2188   PetscCall(DMSetType(*dm, DMPLEX));
2189   PetscCall(DMPlexCreateHexCylinderMesh_Internal(*dm, periodicZ));
2190   PetscFunctionReturn(PETSC_SUCCESS);
2191 }
2192 
2193 static PetscErrorCode DMPlexCreateWedgeCylinderMesh_Internal(DM dm, PetscInt n, PetscBool interpolate)
2194 {
2195   const PetscInt dim = 3;
2196   PetscInt       numCells, numVertices, v;
2197   PetscMPIInt    rank;
2198 
2199   PetscFunctionBegin;
2200   PetscCheck(n >= 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %" PetscInt_FMT " cannot be negative", n);
2201   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
2202   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2203   PetscCall(DMSetDimension(dm, dim));
2204   /* Must create the celltype label here so that we do not automatically try to compute the types */
2205   PetscCall(DMCreateLabel(dm, "celltype"));
2206   /* Create topology */
2207   {
2208     PetscInt cone[6], c;
2209 
2210     numCells    = rank == 0 ? n : 0;
2211     numVertices = rank == 0 ? 2 * (n + 1) : 0;
2212     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
2213     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
2214     PetscCall(DMSetUp(dm));
2215     for (c = 0; c < numCells; c++) {
2216       cone[0] = c + n * 1;
2217       cone[1] = (c + 1) % n + n * 1;
2218       cone[2] = 0 + 3 * n;
2219       cone[3] = c + n * 2;
2220       cone[4] = (c + 1) % n + n * 2;
2221       cone[5] = 1 + 3 * n;
2222       PetscCall(DMPlexSetCone(dm, c, cone));
2223       PetscCall(DMPlexSetCellType(dm, c, DM_POLYTOPE_TRI_PRISM_TENSOR));
2224     }
2225     PetscCall(DMPlexSymmetrize(dm));
2226     PetscCall(DMPlexStratify(dm));
2227   }
2228   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetCellType(dm, v, DM_POLYTOPE_POINT));
2229   /* Create cylinder geometry */
2230   {
2231     Vec          coordinates;
2232     PetscSection coordSection;
2233     PetscScalar *coords;
2234     PetscInt     coordSize, c;
2235 
2236     /* Build coordinates */
2237     PetscCall(DMGetCoordinateSection(dm, &coordSection));
2238     PetscCall(PetscSectionSetNumFields(coordSection, 1));
2239     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
2240     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
2241     for (v = numCells; v < numCells + numVertices; ++v) {
2242       PetscCall(PetscSectionSetDof(coordSection, v, dim));
2243       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
2244     }
2245     PetscCall(PetscSectionSetUp(coordSection));
2246     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2247     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2248     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2249     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2250     PetscCall(VecSetBlockSize(coordinates, dim));
2251     PetscCall(VecSetType(coordinates, VECSTANDARD));
2252     PetscCall(VecGetArray(coordinates, &coords));
2253     for (c = 0; c < numCells; c++) {
2254       coords[(c + 0 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
2255       coords[(c + 0 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
2256       coords[(c + 0 * n) * dim + 2] = 1.0;
2257       coords[(c + 1 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
2258       coords[(c + 1 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
2259       coords[(c + 1 * n) * dim + 2] = 0.0;
2260     }
2261     if (rank == 0) {
2262       coords[(2 * n + 0) * dim + 0] = 0.0;
2263       coords[(2 * n + 0) * dim + 1] = 0.0;
2264       coords[(2 * n + 0) * dim + 2] = 1.0;
2265       coords[(2 * n + 1) * dim + 0] = 0.0;
2266       coords[(2 * n + 1) * dim + 1] = 0.0;
2267       coords[(2 * n + 1) * dim + 2] = 0.0;
2268     }
2269     PetscCall(VecRestoreArray(coordinates, &coords));
2270     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2271     PetscCall(VecDestroy(&coordinates));
2272   }
2273   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
2274   /* Interpolate */
2275   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2276   PetscFunctionReturn(PETSC_SUCCESS);
2277 }
2278 
2279 /*@
2280   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
2281 
2282   Collective
2283 
2284   Input Parameters:
2285 + comm        - The communicator for the `DM` object
2286 . n           - The number of wedges around the origin
2287 - interpolate - Create edges and faces
2288 
2289   Output Parameter:
2290 . dm - The `DM` object
2291 
2292   Level: beginner
2293 
2294 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2295 @*/
2296 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
2297 {
2298   PetscFunctionBegin;
2299   PetscAssertPointer(dm, 4);
2300   PetscCall(DMCreate(comm, dm));
2301   PetscCall(DMSetType(*dm, DMPLEX));
2302   PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(*dm, n, interpolate));
2303   PetscFunctionReturn(PETSC_SUCCESS);
2304 }
2305 
2306 static inline PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2307 {
2308   PetscReal prod = 0.0;
2309   PetscInt  i;
2310   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
2311   return PetscSqrtReal(prod);
2312 }
2313 static inline PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2314 {
2315   PetscReal prod = 0.0;
2316   PetscInt  i;
2317   for (i = 0; i < dim; ++i) prod += x[i] * y[i];
2318   return prod;
2319 }
2320 
2321 /* The first constant is the sphere radius */
2322 static void snapToSphere(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
2323 {
2324   PetscReal r     = PetscRealPart(constants[0]);
2325   PetscReal norm2 = 0.0, fac;
2326   PetscInt  n     = uOff[1] - uOff[0], d;
2327 
2328   for (d = 0; d < n; ++d) norm2 += PetscSqr(PetscRealPart(u[d]));
2329   fac = r / PetscSqrtReal(norm2);
2330   for (d = 0; d < n; ++d) f0[d] = u[d] * fac;
2331 }
2332 
2333 static PetscErrorCode DMPlexCreateSphereMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, PetscReal R)
2334 {
2335   const PetscInt embedDim = dim + 1;
2336   PetscSection   coordSection;
2337   Vec            coordinates;
2338   PetscScalar   *coords;
2339   PetscReal     *coordsIn;
2340   PetscInt       numCells, numEdges, numVerts = 0, firstVertex = 0, v, firstEdge, coordSize, d, c, e;
2341   PetscMPIInt    rank;
2342 
2343   PetscFunctionBegin;
2344   PetscValidLogicalCollectiveBool(dm, simplex, 3);
2345   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
2346   PetscCall(DMSetDimension(dm, dim));
2347   PetscCall(DMSetCoordinateDim(dm, dim + 1));
2348   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2349   switch (dim) {
2350   case 1:
2351     numCells = 16;
2352     numVerts = numCells;
2353 
2354     // Build Topology
2355     PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2356     for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2357     PetscCall(DMSetUp(dm));
2358     for (PetscInt c = 0; c < numCells; ++c) {
2359       PetscInt cone[2];
2360 
2361       cone[0] = c + numCells;
2362       cone[1] = (c + 1) % numVerts + numCells;
2363       PetscCall(DMPlexSetCone(dm, c, cone));
2364     }
2365     PetscCall(DMPlexSymmetrize(dm));
2366     PetscCall(DMPlexStratify(dm));
2367     PetscCall(PetscMalloc1(numVerts * embedDim, &coordsIn));
2368     for (PetscInt v = 0; v < numVerts; ++v) {
2369       const PetscReal rad = 2. * PETSC_PI * v / numVerts;
2370 
2371       coordsIn[v * embedDim + 0] = PetscCosReal(rad);
2372       coordsIn[v * embedDim + 1] = PetscSinReal(rad);
2373     }
2374     break;
2375   case 2:
2376     if (simplex) {
2377       const PetscReal radius    = PetscSqrtReal(1 + PETSC_PHI * PETSC_PHI) / (1.0 + PETSC_PHI);
2378       const PetscReal edgeLen   = 2.0 / (1.0 + PETSC_PHI) * (R / radius);
2379       const PetscInt  degree    = 5;
2380       PetscReal       vertex[3] = {0.0, 1.0 / (1.0 + PETSC_PHI), PETSC_PHI / (1.0 + PETSC_PHI)};
2381       PetscInt        s[3]      = {1, 1, 1};
2382       PetscInt        cone[3];
2383       PetscInt       *graph, p, i, j, k;
2384 
2385       vertex[0] *= R / radius;
2386       vertex[1] *= R / radius;
2387       vertex[2] *= R / radius;
2388       numCells    = rank == 0 ? 20 : 0;
2389       numVerts    = rank == 0 ? 12 : 0;
2390       firstVertex = numCells;
2391       /* Use icosahedron, which for a R-sphere has coordinates which are all cyclic permutations of
2392 
2393            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
2394 
2395          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2396          length is then given by 2/(1+\phi) = 2 * 0.38197 = 0.76393.
2397       */
2398       /* Construct vertices */
2399       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2400       if (rank == 0) {
2401         for (p = 0, i = 0; p < embedDim; ++p) {
2402           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2403             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2404               for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertex[(d + p) % embedDim];
2405               ++i;
2406             }
2407           }
2408         }
2409       }
2410       /* Construct graph */
2411       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2412       for (i = 0; i < numVerts; ++i) {
2413         for (j = 0, k = 0; j < numVerts; ++j) {
2414           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2415             graph[i * numVerts + j] = 1;
2416             ++k;
2417           }
2418         }
2419         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid icosahedron, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2420       }
2421       /* Build Topology */
2422       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2423       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2424       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2425       /* Cells */
2426       for (i = 0, c = 0; i < numVerts; ++i) {
2427         for (j = 0; j < i; ++j) {
2428           for (k = 0; k < j; ++k) {
2429             if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i]) {
2430               cone[0] = firstVertex + i;
2431               cone[1] = firstVertex + j;
2432               cone[2] = firstVertex + k;
2433               /* Check orientation */
2434               {
2435                 const PetscInt epsilon[3][3][3] = {
2436                   {{0, 0, 0},  {0, 0, 1},  {0, -1, 0}},
2437                   {{0, 0, -1}, {0, 0, 0},  {1, 0, 0} },
2438                   {{0, 1, 0},  {-1, 0, 0}, {0, 0, 0} }
2439                 };
2440                 PetscReal normal[3];
2441                 PetscInt  e, f;
2442 
2443                 for (d = 0; d < embedDim; ++d) {
2444                   normal[d] = 0.0;
2445                   for (e = 0; e < embedDim; ++e) {
2446                     for (f = 0; f < embedDim; ++f) normal[d] += epsilon[d][e][f] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]);
2447                   }
2448                 }
2449                 if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2450                   PetscInt tmp = cone[1];
2451                   cone[1]      = cone[2];
2452                   cone[2]      = tmp;
2453                 }
2454               }
2455               PetscCall(DMPlexSetCone(dm, c++, cone));
2456             }
2457           }
2458         }
2459       }
2460       PetscCall(DMPlexSymmetrize(dm));
2461       PetscCall(DMPlexStratify(dm));
2462       PetscCall(PetscFree(graph));
2463     } else {
2464       /*
2465         12-21--13
2466          |     |
2467         25  4  24
2468          |     |
2469   12-25--9-16--8-24--13
2470    |     |     |     |
2471   23  5 17  0 15  3  22
2472    |     |     |     |
2473   10-20--6-14--7-19--11
2474          |     |
2475         20  1  19
2476          |     |
2477         10-18--11
2478          |     |
2479         23  2  22
2480          |     |
2481         12-21--13
2482        */
2483       PetscInt cone[4], ornt[4];
2484 
2485       numCells    = rank == 0 ? 6 : 0;
2486       numEdges    = rank == 0 ? 12 : 0;
2487       numVerts    = rank == 0 ? 8 : 0;
2488       firstVertex = numCells;
2489       firstEdge   = numCells + numVerts;
2490       /* Build Topology */
2491       PetscCall(DMPlexSetChart(dm, 0, numCells + numEdges + numVerts));
2492       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 4));
2493       for (e = firstEdge; e < firstEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
2494       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2495       if (rank == 0) {
2496         /* Cell 0 */
2497         cone[0] = 14;
2498         cone[1] = 15;
2499         cone[2] = 16;
2500         cone[3] = 17;
2501         PetscCall(DMPlexSetCone(dm, 0, cone));
2502         ornt[0] = 0;
2503         ornt[1] = 0;
2504         ornt[2] = 0;
2505         ornt[3] = 0;
2506         PetscCall(DMPlexSetConeOrientation(dm, 0, ornt));
2507         /* Cell 1 */
2508         cone[0] = 18;
2509         cone[1] = 19;
2510         cone[2] = 14;
2511         cone[3] = 20;
2512         PetscCall(DMPlexSetCone(dm, 1, cone));
2513         ornt[0] = 0;
2514         ornt[1] = 0;
2515         ornt[2] = -1;
2516         ornt[3] = 0;
2517         PetscCall(DMPlexSetConeOrientation(dm, 1, ornt));
2518         /* Cell 2 */
2519         cone[0] = 21;
2520         cone[1] = 22;
2521         cone[2] = 18;
2522         cone[3] = 23;
2523         PetscCall(DMPlexSetCone(dm, 2, cone));
2524         ornt[0] = 0;
2525         ornt[1] = 0;
2526         ornt[2] = -1;
2527         ornt[3] = 0;
2528         PetscCall(DMPlexSetConeOrientation(dm, 2, ornt));
2529         /* Cell 3 */
2530         cone[0] = 19;
2531         cone[1] = 22;
2532         cone[2] = 24;
2533         cone[3] = 15;
2534         PetscCall(DMPlexSetCone(dm, 3, cone));
2535         ornt[0] = -1;
2536         ornt[1] = -1;
2537         ornt[2] = 0;
2538         ornt[3] = -1;
2539         PetscCall(DMPlexSetConeOrientation(dm, 3, ornt));
2540         /* Cell 4 */
2541         cone[0] = 16;
2542         cone[1] = 24;
2543         cone[2] = 21;
2544         cone[3] = 25;
2545         PetscCall(DMPlexSetCone(dm, 4, cone));
2546         ornt[0] = -1;
2547         ornt[1] = -1;
2548         ornt[2] = -1;
2549         ornt[3] = 0;
2550         PetscCall(DMPlexSetConeOrientation(dm, 4, ornt));
2551         /* Cell 5 */
2552         cone[0] = 20;
2553         cone[1] = 17;
2554         cone[2] = 25;
2555         cone[3] = 23;
2556         PetscCall(DMPlexSetCone(dm, 5, cone));
2557         ornt[0] = -1;
2558         ornt[1] = -1;
2559         ornt[2] = -1;
2560         ornt[3] = -1;
2561         PetscCall(DMPlexSetConeOrientation(dm, 5, ornt));
2562         /* Edges */
2563         cone[0] = 6;
2564         cone[1] = 7;
2565         PetscCall(DMPlexSetCone(dm, 14, cone));
2566         cone[0] = 7;
2567         cone[1] = 8;
2568         PetscCall(DMPlexSetCone(dm, 15, cone));
2569         cone[0] = 8;
2570         cone[1] = 9;
2571         PetscCall(DMPlexSetCone(dm, 16, cone));
2572         cone[0] = 9;
2573         cone[1] = 6;
2574         PetscCall(DMPlexSetCone(dm, 17, cone));
2575         cone[0] = 10;
2576         cone[1] = 11;
2577         PetscCall(DMPlexSetCone(dm, 18, cone));
2578         cone[0] = 11;
2579         cone[1] = 7;
2580         PetscCall(DMPlexSetCone(dm, 19, cone));
2581         cone[0] = 6;
2582         cone[1] = 10;
2583         PetscCall(DMPlexSetCone(dm, 20, cone));
2584         cone[0] = 12;
2585         cone[1] = 13;
2586         PetscCall(DMPlexSetCone(dm, 21, cone));
2587         cone[0] = 13;
2588         cone[1] = 11;
2589         PetscCall(DMPlexSetCone(dm, 22, cone));
2590         cone[0] = 10;
2591         cone[1] = 12;
2592         PetscCall(DMPlexSetCone(dm, 23, cone));
2593         cone[0] = 13;
2594         cone[1] = 8;
2595         PetscCall(DMPlexSetCone(dm, 24, cone));
2596         cone[0] = 12;
2597         cone[1] = 9;
2598         PetscCall(DMPlexSetCone(dm, 25, cone));
2599       }
2600       PetscCall(DMPlexSymmetrize(dm));
2601       PetscCall(DMPlexStratify(dm));
2602       /* Build coordinates */
2603       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2604       if (rank == 0) {
2605         coordsIn[0 * embedDim + 0] = -R;
2606         coordsIn[0 * embedDim + 1] = R;
2607         coordsIn[0 * embedDim + 2] = -R;
2608         coordsIn[1 * embedDim + 0] = R;
2609         coordsIn[1 * embedDim + 1] = R;
2610         coordsIn[1 * embedDim + 2] = -R;
2611         coordsIn[2 * embedDim + 0] = R;
2612         coordsIn[2 * embedDim + 1] = -R;
2613         coordsIn[2 * embedDim + 2] = -R;
2614         coordsIn[3 * embedDim + 0] = -R;
2615         coordsIn[3 * embedDim + 1] = -R;
2616         coordsIn[3 * embedDim + 2] = -R;
2617         coordsIn[4 * embedDim + 0] = -R;
2618         coordsIn[4 * embedDim + 1] = R;
2619         coordsIn[4 * embedDim + 2] = R;
2620         coordsIn[5 * embedDim + 0] = R;
2621         coordsIn[5 * embedDim + 1] = R;
2622         coordsIn[5 * embedDim + 2] = R;
2623         coordsIn[6 * embedDim + 0] = -R;
2624         coordsIn[6 * embedDim + 1] = -R;
2625         coordsIn[6 * embedDim + 2] = R;
2626         coordsIn[7 * embedDim + 0] = R;
2627         coordsIn[7 * embedDim + 1] = -R;
2628         coordsIn[7 * embedDim + 2] = R;
2629       }
2630     }
2631     break;
2632   case 3:
2633     if (simplex) {
2634       const PetscReal edgeLen         = 1.0 / PETSC_PHI;
2635       PetscReal       vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
2636       PetscReal       vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
2637       PetscReal       vertexC[4]      = {0.5, 0.5 * PETSC_PHI, 0.5 / PETSC_PHI, 0.0};
2638       const PetscInt  degree          = 12;
2639       PetscInt        s[4]            = {1, 1, 1};
2640       PetscInt        evenPerm[12][4] = {
2641         {0, 1, 2, 3},
2642         {0, 2, 3, 1},
2643         {0, 3, 1, 2},
2644         {1, 0, 3, 2},
2645         {1, 2, 0, 3},
2646         {1, 3, 2, 0},
2647         {2, 0, 1, 3},
2648         {2, 1, 3, 0},
2649         {2, 3, 0, 1},
2650         {3, 0, 2, 1},
2651         {3, 1, 0, 2},
2652         {3, 2, 1, 0}
2653       };
2654       PetscInt  cone[4];
2655       PetscInt *graph, p, i, j, k, l;
2656 
2657       vertexA[0] *= R;
2658       vertexA[1] *= R;
2659       vertexA[2] *= R;
2660       vertexA[3] *= R;
2661       vertexB[0] *= R;
2662       vertexB[1] *= R;
2663       vertexB[2] *= R;
2664       vertexB[3] *= R;
2665       vertexC[0] *= R;
2666       vertexC[1] *= R;
2667       vertexC[2] *= R;
2668       vertexC[3] *= R;
2669       numCells    = rank == 0 ? 600 : 0;
2670       numVerts    = rank == 0 ? 120 : 0;
2671       firstVertex = numCells;
2672       /* Use the 600-cell, which for a unit sphere has coordinates which are
2673 
2674            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
2675                (\pm 1,    0,       0,      0)  all cyclic permutations   8
2676            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
2677 
2678          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2679          length is then given by 1/\phi = 0.61803.
2680 
2681          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
2682          http://mathworld.wolfram.com/600-Cell.html
2683       */
2684       /* Construct vertices */
2685       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2686       i = 0;
2687       if (rank == 0) {
2688         for (s[0] = -1; s[0] < 2; s[0] += 2) {
2689           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2690             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2691               for (s[3] = -1; s[3] < 2; s[3] += 2) {
2692                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[d] * vertexA[d];
2693                 ++i;
2694               }
2695             }
2696           }
2697         }
2698         for (p = 0; p < embedDim; ++p) {
2699           s[1] = s[2] = s[3] = 1;
2700           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2701             for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertexB[(d + p) % embedDim];
2702             ++i;
2703           }
2704         }
2705         for (p = 0; p < 12; ++p) {
2706           s[3] = 1;
2707           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2708             for (s[1] = -1; s[1] < 2; s[1] += 2) {
2709               for (s[2] = -1; s[2] < 2; s[2] += 2) {
2710                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[evenPerm[p][d]] * vertexC[evenPerm[p][d]];
2711                 ++i;
2712               }
2713             }
2714           }
2715         }
2716       }
2717       PetscCheck(i == numVerts, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertices %" PetscInt_FMT " != %" PetscInt_FMT, i, numVerts);
2718       /* Construct graph */
2719       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2720       for (i = 0; i < numVerts; ++i) {
2721         for (j = 0, k = 0; j < numVerts; ++j) {
2722           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2723             graph[i * numVerts + j] = 1;
2724             ++k;
2725           }
2726         }
2727         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2728       }
2729       /* Build Topology */
2730       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2731       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2732       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2733       /* Cells */
2734       if (rank == 0) {
2735         for (i = 0, c = 0; i < numVerts; ++i) {
2736           for (j = 0; j < i; ++j) {
2737             for (k = 0; k < j; ++k) {
2738               for (l = 0; l < k; ++l) {
2739                 if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i] && graph[l * numVerts + i] && graph[l * numVerts + j] && graph[l * numVerts + k]) {
2740                   cone[0] = firstVertex + i;
2741                   cone[1] = firstVertex + j;
2742                   cone[2] = firstVertex + k;
2743                   cone[3] = firstVertex + l;
2744                   /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
2745                   {
2746                     const PetscInt epsilon[4][4][4][4] = {
2747                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, -1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 0, 0}, {0, 1, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 1, 0}, {0, -1, 0, 0}, {0, 0, 0, 0}}},
2748 
2749                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}}, {{0, 0, -1, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}}},
2750 
2751                       {{{0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, 0, 0}, {0, -1, 0, 0}}, {{0, 0, 0, -1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 1, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}},
2752 
2753                       {{{0, 0, 0, 0}, {0, 0, -1, 0}, {0, 1, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 1, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}}, {{0, -1, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}} }
2754                     };
2755                     PetscReal normal[4];
2756                     PetscInt  e, f, g;
2757 
2758                     for (d = 0; d < embedDim; ++d) {
2759                       normal[d] = 0.0;
2760                       for (e = 0; e < embedDim; ++e) {
2761                         for (f = 0; f < embedDim; ++f) {
2762                           for (g = 0; g < embedDim; ++g) {
2763                             normal[d] += epsilon[d][e][f][g] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]) * (coordsIn[l * embedDim + f] - coordsIn[i * embedDim + f]);
2764                           }
2765                         }
2766                       }
2767                     }
2768                     if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2769                       PetscInt tmp = cone[1];
2770                       cone[1]      = cone[2];
2771                       cone[2]      = tmp;
2772                     }
2773                   }
2774                   PetscCall(DMPlexSetCone(dm, c++, cone));
2775                 }
2776               }
2777             }
2778           }
2779         }
2780       }
2781       PetscCall(DMPlexSymmetrize(dm));
2782       PetscCall(DMPlexStratify(dm));
2783       PetscCall(PetscFree(graph));
2784     }
2785     break;
2786   default:
2787     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unsupported dimension for sphere: %" PetscInt_FMT, dim);
2788   }
2789   /* Create coordinates */
2790   PetscCall(DMGetCoordinateSection(dm, &coordSection));
2791   PetscCall(PetscSectionSetNumFields(coordSection, 1));
2792   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, embedDim));
2793   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVerts));
2794   for (v = firstVertex; v < firstVertex + numVerts; ++v) {
2795     PetscCall(PetscSectionSetDof(coordSection, v, embedDim));
2796     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, embedDim));
2797   }
2798   PetscCall(PetscSectionSetUp(coordSection));
2799   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2800   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2801   PetscCall(VecSetBlockSize(coordinates, embedDim));
2802   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2803   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2804   PetscCall(VecSetType(coordinates, VECSTANDARD));
2805   PetscCall(VecGetArray(coordinates, &coords));
2806   for (v = 0; v < numVerts; ++v)
2807     for (d = 0; d < embedDim; ++d) coords[v * embedDim + d] = coordsIn[v * embedDim + d];
2808   PetscCall(VecRestoreArray(coordinates, &coords));
2809   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2810   PetscCall(VecDestroy(&coordinates));
2811   PetscCall(PetscFree(coordsIn));
2812   {
2813     DM          cdm;
2814     PetscDS     cds;
2815     PetscScalar c = R;
2816 
2817     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_TRUE, snapToSphere));
2818     PetscCall(DMGetCoordinateDM(dm, &cdm));
2819     PetscCall(DMGetDS(cdm, &cds));
2820     PetscCall(PetscDSSetConstants(cds, 1, &c));
2821   }
2822   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
2823   /* Wait for coordinate creation before doing in-place modification */
2824   if (simplex) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2825   PetscFunctionReturn(PETSC_SUCCESS);
2826 }
2827 
2828 typedef void (*TPSEvaluateFunc)(const PetscReal[], PetscReal *, PetscReal[], PetscReal (*)[3]);
2829 
2830 /*
2831  The Schwarz P implicit surface is
2832 
2833      f(x) = cos(x0) + cos(x1) + cos(x2) = 0
2834 */
2835 static void TPSEvaluate_SchwarzP(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2836 {
2837   PetscReal c[3] = {PetscCosReal(y[0] * PETSC_PI), PetscCosReal(y[1] * PETSC_PI), PetscCosReal(y[2] * PETSC_PI)};
2838   PetscReal g[3] = {-PetscSinReal(y[0] * PETSC_PI), -PetscSinReal(y[1] * PETSC_PI), -PetscSinReal(y[2] * PETSC_PI)};
2839   f[0]           = c[0] + c[1] + c[2];
2840   for (PetscInt i = 0; i < 3; i++) {
2841     grad[i] = PETSC_PI * g[i];
2842     for (PetscInt j = 0; j < 3; j++) hess[i][j] = (i == j) ? -PetscSqr(PETSC_PI) * c[i] : 0.;
2843   }
2844 }
2845 
2846 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2847 static PetscErrorCode TPSExtrudeNormalFunc_SchwarzP(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2848 {
2849   for (PetscInt i = 0; i < 3; i++) u[i] = -PETSC_PI * PetscSinReal(x[i] * PETSC_PI);
2850   return PETSC_SUCCESS;
2851 }
2852 
2853 /*
2854  The Gyroid implicit surface is
2855 
2856  f(x,y,z) = sin(pi * x) * cos (pi * (y + 1/2))  + sin(pi * (y + 1/2)) * cos(pi * (z + 1/4)) + sin(pi * (z + 1/4)) * cos(pi * x)
2857 
2858 */
2859 static void TPSEvaluate_Gyroid(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2860 {
2861   PetscReal s[3] = {PetscSinReal(PETSC_PI * y[0]), PetscSinReal(PETSC_PI * (y[1] + .5)), PetscSinReal(PETSC_PI * (y[2] + .25))};
2862   PetscReal c[3] = {PetscCosReal(PETSC_PI * y[0]), PetscCosReal(PETSC_PI * (y[1] + .5)), PetscCosReal(PETSC_PI * (y[2] + .25))};
2863   f[0]           = s[0] * c[1] + s[1] * c[2] + s[2] * c[0];
2864   grad[0]        = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2865   grad[1]        = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2866   grad[2]        = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2867   hess[0][0]     = -PetscSqr(PETSC_PI) * (s[0] * c[1] + s[2] * c[0]);
2868   hess[0][1]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2869   hess[0][2]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2870   hess[1][0]     = -PetscSqr(PETSC_PI) * (s[1] * c[2] + s[0] * c[1]);
2871   hess[1][1]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2872   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2873   hess[2][0]     = -PetscSqr(PETSC_PI) * (s[2] * c[0] + s[1] * c[2]);
2874   hess[2][1]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2875   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2876 }
2877 
2878 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2879 static PetscErrorCode TPSExtrudeNormalFunc_Gyroid(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2880 {
2881   PetscReal s[3] = {PetscSinReal(PETSC_PI * x[0]), PetscSinReal(PETSC_PI * (x[1] + .5)), PetscSinReal(PETSC_PI * (x[2] + .25))};
2882   PetscReal c[3] = {PetscCosReal(PETSC_PI * x[0]), PetscCosReal(PETSC_PI * (x[1] + .5)), PetscCosReal(PETSC_PI * (x[2] + .25))};
2883   u[0]           = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2884   u[1]           = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2885   u[2]           = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2886   return PETSC_SUCCESS;
2887 }
2888 
2889 /*
2890    We wish to solve
2891 
2892          min_y || y - x ||^2  subject to f(y) = 0
2893 
2894    Let g(y) = grad(f).  The minimization problem is equivalent to asking to satisfy
2895    f(y) = 0 and (y-x) is parallel to g(y).  We do this by using Householder QR to obtain a basis for the
2896    tangent space and ask for both components in the tangent space to be zero.
2897 
2898    Take g to be a column vector and compute the "full QR" factorization Q R = g,
2899    where Q = I - 2 n n^T is a symmetric orthogonal matrix.
2900    The first column of Q is parallel to g so the remaining two columns span the null space.
2901    Let Qn = Q[:,1:] be those remaining columns.  Then Qn Qn^T is an orthogonal projector into the tangent space.
2902    Since Q is symmetric, this is equivalent to multiplying by Q and taking the last two entries.
2903    In total, we have a system of 3 equations in 3 unknowns:
2904 
2905      f(y) = 0                       1 equation
2906      Qn^T (y - x) = 0               2 equations
2907 
2908    Here, we compute the residual and Jacobian of this system.
2909 */
2910 static void TPSNearestPointResJac(TPSEvaluateFunc feval, const PetscScalar x[], const PetscScalar y[], PetscScalar res[], PetscScalar J[])
2911 {
2912   PetscReal yreal[3] = {PetscRealPart(y[0]), PetscRealPart(y[1]), PetscRealPart(y[2])};
2913   PetscReal d[3]     = {PetscRealPart(y[0] - x[0]), PetscRealPart(y[1] - x[1]), PetscRealPart(y[2] - x[2])};
2914   PetscReal f, grad[3], n[3], norm, norm_y[3], nd, nd_y[3], sign;
2915   PetscReal n_y[3][3] = {
2916     {0, 0, 0},
2917     {0, 0, 0},
2918     {0, 0, 0}
2919   };
2920 
2921   feval(yreal, &f, grad, n_y);
2922 
2923   for (PetscInt i = 0; i < 3; i++) n[i] = grad[i];
2924   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2925   for (PetscInt i = 0; i < 3; i++) norm_y[i] = 1. / norm * n[i] * n_y[i][i];
2926 
2927   // Define the Householder reflector
2928   sign = n[0] >= 0 ? 1. : -1.;
2929   n[0] += norm * sign;
2930   for (PetscInt i = 0; i < 3; i++) n_y[0][i] += norm_y[i] * sign;
2931 
2932   norm      = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2933   norm_y[0] = 1. / norm * (n[0] * n_y[0][0]);
2934   norm_y[1] = 1. / norm * (n[0] * n_y[0][1] + n[1] * n_y[1][1]);
2935   norm_y[2] = 1. / norm * (n[0] * n_y[0][2] + n[2] * n_y[2][2]);
2936 
2937   for (PetscInt i = 0; i < 3; i++) {
2938     n[i] /= norm;
2939     for (PetscInt j = 0; j < 3; j++) {
2940       // note that n[i] is n_old[i]/norm when executing the code below
2941       n_y[i][j] = n_y[i][j] / norm - n[i] / norm * norm_y[j];
2942     }
2943   }
2944 
2945   nd = n[0] * d[0] + n[1] * d[1] + n[2] * d[2];
2946   for (PetscInt i = 0; i < 3; i++) nd_y[i] = n[i] + n_y[0][i] * d[0] + n_y[1][i] * d[1] + n_y[2][i] * d[2];
2947 
2948   res[0] = f;
2949   res[1] = d[1] - 2 * n[1] * nd;
2950   res[2] = d[2] - 2 * n[2] * nd;
2951   // J[j][i] is J_{ij} (column major)
2952   for (PetscInt j = 0; j < 3; j++) {
2953     J[0 + j * 3] = grad[j];
2954     J[1 + j * 3] = (j == 1) * 1. - 2 * (n_y[1][j] * nd + n[1] * nd_y[j]);
2955     J[2 + j * 3] = (j == 2) * 1. - 2 * (n_y[2][j] * nd + n[2] * nd_y[j]);
2956   }
2957 }
2958 
2959 /*
2960    Project x to the nearest point on the implicit surface using Newton's method.
2961 */
2962 static PetscErrorCode TPSNearestPoint(TPSEvaluateFunc feval, PetscScalar x[])
2963 {
2964   PetscScalar y[3] = {x[0], x[1], x[2]}; // Initial guess
2965 
2966   PetscFunctionBegin;
2967   for (PetscInt iter = 0; iter < 10; iter++) {
2968     PetscScalar res[3], J[9];
2969     PetscReal   resnorm;
2970     TPSNearestPointResJac(feval, x, y, res, J);
2971     resnorm = PetscSqrtReal(PetscSqr(PetscRealPart(res[0])) + PetscSqr(PetscRealPart(res[1])) + PetscSqr(PetscRealPart(res[2])));
2972     if (0) { // Turn on this monitor if you need to confirm quadratic convergence
2973       PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%" PetscInt_FMT "] res [%g %g %g]\n", iter, (double)PetscRealPart(res[0]), (double)PetscRealPart(res[1]), (double)PetscRealPart(res[2])));
2974     }
2975     if (resnorm < PETSC_SMALL) break;
2976 
2977     // Take the Newton step
2978     PetscCall(PetscKernel_A_gets_inverse_A_3(J, 0., PETSC_FALSE, NULL));
2979     PetscKernel_v_gets_v_minus_A_times_w_3(y, J, res);
2980   }
2981   for (PetscInt i = 0; i < 3; i++) x[i] = y[i];
2982   PetscFunctionReturn(PETSC_SUCCESS);
2983 }
2984 
2985 const char *const DMPlexTPSTypes[] = {"SCHWARZ_P", "GYROID", "DMPlexTPSType", "DMPLEX_TPS_", NULL};
2986 
2987 static PetscErrorCode DMPlexCreateTPSMesh_Internal(DM dm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness)
2988 {
2989   PetscMPIInt rank;
2990   PetscInt    topoDim = 2, spaceDim = 3, numFaces = 0, numVertices = 0, numEdges = 0;
2991   PetscInt(*edges)[2] = NULL, *edgeSets = NULL;
2992   PetscInt            *cells_flat = NULL;
2993   PetscReal           *vtxCoords  = NULL;
2994   TPSEvaluateFunc      evalFunc   = NULL;
2995   PetscSimplePointFunc normalFunc = NULL;
2996   DMLabel              label;
2997 
2998   PetscFunctionBegin;
2999   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
3000   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3001   PetscCheck((layers != 0) ^ (thickness == 0.), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Layers %" PetscInt_FMT " must be nonzero iff thickness %g is nonzero", layers, (double)thickness);
3002   switch (tpstype) {
3003   case DMPLEX_TPS_SCHWARZ_P:
3004     PetscCheck(!periodic || (periodic[0] == DM_BOUNDARY_NONE && periodic[1] == DM_BOUNDARY_NONE && periodic[2] == DM_BOUNDARY_NONE), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Schwarz P does not support periodic meshes");
3005     if (rank == 0) {
3006       PetscInt(*cells)[6][4][4] = NULL; // [junction, junction-face, cell, conn]
3007       PetscInt  Njunctions = 0, Ncuts = 0, Npipes[3], vcount;
3008       PetscReal L = 1;
3009 
3010       Npipes[0]   = (extent[0] + 1) * extent[1] * extent[2];
3011       Npipes[1]   = extent[0] * (extent[1] + 1) * extent[2];
3012       Npipes[2]   = extent[0] * extent[1] * (extent[2] + 1);
3013       Njunctions  = extent[0] * extent[1] * extent[2];
3014       Ncuts       = 2 * (extent[0] * extent[1] + extent[1] * extent[2] + extent[2] * extent[0]);
3015       numVertices = 4 * (Npipes[0] + Npipes[1] + Npipes[2]) + 8 * Njunctions;
3016       PetscCall(PetscMalloc1(3 * numVertices, &vtxCoords));
3017       PetscCall(PetscMalloc1(Njunctions, &cells));
3018       PetscCall(PetscMalloc1(Ncuts * 4, &edges));
3019       PetscCall(PetscMalloc1(Ncuts * 4, &edgeSets));
3020       // x-normal pipes
3021       vcount = 0;
3022       for (PetscInt i = 0; i < extent[0] + 1; i++) {
3023         for (PetscInt j = 0; j < extent[1]; j++) {
3024           for (PetscInt k = 0; k < extent[2]; k++) {
3025             for (PetscInt l = 0; l < 4; l++) {
3026               vtxCoords[vcount++] = (2 * i - 1) * L;
3027               vtxCoords[vcount++] = 2 * j * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3028               vtxCoords[vcount++] = 2 * k * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3029             }
3030           }
3031         }
3032       }
3033       // y-normal pipes
3034       for (PetscInt i = 0; i < extent[0]; i++) {
3035         for (PetscInt j = 0; j < extent[1] + 1; j++) {
3036           for (PetscInt k = 0; k < extent[2]; k++) {
3037             for (PetscInt l = 0; l < 4; l++) {
3038               vtxCoords[vcount++] = 2 * i * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3039               vtxCoords[vcount++] = (2 * j - 1) * L;
3040               vtxCoords[vcount++] = 2 * k * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3041             }
3042           }
3043         }
3044       }
3045       // z-normal pipes
3046       for (PetscInt i = 0; i < extent[0]; i++) {
3047         for (PetscInt j = 0; j < extent[1]; j++) {
3048           for (PetscInt k = 0; k < extent[2] + 1; k++) {
3049             for (PetscInt l = 0; l < 4; l++) {
3050               vtxCoords[vcount++] = 2 * i * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3051               vtxCoords[vcount++] = 2 * j * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3052               vtxCoords[vcount++] = (2 * k - 1) * L;
3053             }
3054           }
3055         }
3056       }
3057       // junctions
3058       for (PetscInt i = 0; i < extent[0]; i++) {
3059         for (PetscInt j = 0; j < extent[1]; j++) {
3060           for (PetscInt k = 0; k < extent[2]; k++) {
3061             const PetscInt J = (i * extent[1] + j) * extent[2] + k, Jvoff = (Npipes[0] + Npipes[1] + Npipes[2]) * 4 + J * 8;
3062             PetscCheck(vcount / 3 == Jvoff, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected vertex count");
3063             for (PetscInt ii = 0; ii < 2; ii++) {
3064               for (PetscInt jj = 0; jj < 2; jj++) {
3065                 for (PetscInt kk = 0; kk < 2; kk++) {
3066                   double Ls           = (1 - sqrt(2) / 4) * L;
3067                   vtxCoords[vcount++] = 2 * i * L + (2 * ii - 1) * Ls;
3068                   vtxCoords[vcount++] = 2 * j * L + (2 * jj - 1) * Ls;
3069                   vtxCoords[vcount++] = 2 * k * L + (2 * kk - 1) * Ls;
3070                 }
3071               }
3072             }
3073             const PetscInt jfaces[3][2][4] = {
3074               {{3, 1, 0, 2}, {7, 5, 4, 6}}, // x-aligned
3075               {{5, 4, 0, 1}, {7, 6, 2, 3}}, // y-aligned
3076               {{6, 2, 0, 4}, {7, 3, 1, 5}}  // z-aligned
3077             };
3078             const PetscInt pipe_lo[3] = {// vertex numbers of pipes
3079                                          ((i * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + Npipes[0] + Npipes[1]) * 4};
3080             const PetscInt pipe_hi[3] = {// vertex numbers of pipes
3081                                          (((i + 1) * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j + 1) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + 1 + Npipes[0] + Npipes[1]) * 4};
3082             for (PetscInt dir = 0; dir < 3; dir++) { // x,y,z
3083               const PetscInt ijk[3] = {i, j, k};
3084               for (PetscInt l = 0; l < 4; l++) { // rotations
3085                 cells[J][dir * 2 + 0][l][0] = pipe_lo[dir] + l;
3086                 cells[J][dir * 2 + 0][l][1] = Jvoff + jfaces[dir][0][l];
3087                 cells[J][dir * 2 + 0][l][2] = Jvoff + jfaces[dir][0][(l - 1 + 4) % 4];
3088                 cells[J][dir * 2 + 0][l][3] = pipe_lo[dir] + (l - 1 + 4) % 4;
3089                 cells[J][dir * 2 + 1][l][0] = Jvoff + jfaces[dir][1][l];
3090                 cells[J][dir * 2 + 1][l][1] = pipe_hi[dir] + l;
3091                 cells[J][dir * 2 + 1][l][2] = pipe_hi[dir] + (l - 1 + 4) % 4;
3092                 cells[J][dir * 2 + 1][l][3] = Jvoff + jfaces[dir][1][(l - 1 + 4) % 4];
3093                 if (ijk[dir] == 0) {
3094                   edges[numEdges][0] = pipe_lo[dir] + l;
3095                   edges[numEdges][1] = pipe_lo[dir] + (l + 1) % 4;
3096                   edgeSets[numEdges] = dir * 2 + 1;
3097                   numEdges++;
3098                 }
3099                 if (ijk[dir] + 1 == extent[dir]) {
3100                   edges[numEdges][0] = pipe_hi[dir] + l;
3101                   edges[numEdges][1] = pipe_hi[dir] + (l + 1) % 4;
3102                   edgeSets[numEdges] = dir * 2 + 2;
3103                   numEdges++;
3104                 }
3105               }
3106             }
3107           }
3108         }
3109       }
3110       PetscCheck(numEdges == Ncuts * 4, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Edge count %" PetscInt_FMT " incompatible with number of cuts %" PetscInt_FMT, numEdges, Ncuts);
3111       numFaces   = 24 * Njunctions;
3112       cells_flat = cells[0][0][0];
3113     }
3114     evalFunc   = TPSEvaluate_SchwarzP;
3115     normalFunc = TPSExtrudeNormalFunc_SchwarzP;
3116     break;
3117   case DMPLEX_TPS_GYROID:
3118     if (rank == 0) {
3119       // This is a coarse mesh approximation of the gyroid shifted to being the zero of the level set
3120       //
3121       //     sin(pi*x)*cos(pi*(y+1/2)) + sin(pi*(y+1/2))*cos(pi*(z+1/4)) + sin(pi*(z+1/4))*cos(x)
3122       //
3123       // on the cell [0,2]^3.
3124       //
3125       // Think about dividing that cell into four columns, and focus on the column [0,1]x[0,1]x[0,2].
3126       // If you looked at the gyroid in that column at different slices of z you would see that it kind of spins
3127       // like a boomerang:
3128       //
3129       //     z = 0          z = 1/4        z = 1/2        z = 3/4     //
3130       //     -----          -------        -------        -------     //
3131       //                                                              //
3132       //     +       +      +       +      +       +      +   \   +   //
3133       //      \                                   /            \      //
3134       //       \            `-_   _-'            /              }     //
3135       //        *-_            `-'            _-'              /      //
3136       //     +     `-+      +       +      +-'     +      +   /   +   //
3137       //                                                              //
3138       //                                                              //
3139       //     z = 1          z = 5/4        z = 3/2        z = 7/4     //
3140       //     -----          -------        -------        -------     //
3141       //                                                              //
3142       //     +-_     +      +       +      +     _-+      +   /   +   //
3143       //        `-_            _-_            _-`            /        //
3144       //           \        _-'   `-_        /              {         //
3145       //            \                       /                \        //
3146       //     +       +      +       +      +       +      +   \   +   //
3147       //
3148       //
3149       // This course mesh approximates each of these slices by two line segments,
3150       // and then connects the segments in consecutive layers with quadrilateral faces.
3151       // All of the end points of the segments are multiples of 1/4 except for the
3152       // point * in the picture for z = 0 above and the similar points in other layers.
3153       // That point is at (gamma, gamma, 0), where gamma is calculated below.
3154       //
3155       // The column  [1,2]x[1,2]x[0,2] looks the same as this column;
3156       // The columns [1,2]x[0,1]x[0,2] and [0,1]x[1,2]x[0,2] are mirror images.
3157       //
3158       // As for how this method turned into the names given to the vertices:
3159       // that was not systematic, it was just the way it worked out in my handwritten notes.
3160 
3161       PetscInt facesPerBlock = 64;
3162       PetscInt vertsPerBlock = 56;
3163       PetscInt extentPlus[3];
3164       PetscInt numBlocks, numBlocksPlus;
3165       const PetscInt A = 0, B = 1, C = 2, D = 3, E = 4, F = 5, G = 6, H = 7, II = 8, J = 9, K = 10, L = 11, M = 12, N = 13, O = 14, P = 15, Q = 16, R = 17, S = 18, T = 19, U = 20, V = 21, W = 22, X = 23, Y = 24, Z = 25, Ap = 26, Bp = 27, Cp = 28, Dp = 29, Ep = 30, Fp = 31, Gp = 32, Hp = 33, Ip = 34, Jp = 35, Kp = 36, Lp = 37, Mp = 38, Np = 39, Op = 40, Pp = 41, Qp = 42, Rp = 43, Sp = 44, Tp = 45, Up = 46, Vp = 47, Wp = 48, Xp = 49, Yp = 50, Zp = 51, Aq = 52, Bq = 53, Cq = 54, Dq = 55;
3166       const PetscInt pattern[64][4] = {
3167   /* face to vertex within the coarse discretization of a single gyroid block */
3168   /* layer 0 */
3169         {A,           C,           K,           G          },
3170         {C,           B,           II,          K          },
3171         {D,           A,           H,           L          },
3172         {B + 56 * 1,  D,           L,           J          },
3173         {E,           B + 56 * 1,  J,           N          },
3174         {A + 56 * 2,  E,           N,           H + 56 * 2 },
3175         {F,           A + 56 * 2,  G + 56 * 2,  M          },
3176         {B,           F,           M,           II         },
3177  /* layer 1 */
3178         {G,           K,           Q,           O          },
3179         {K,           II,          P,           Q          },
3180         {L,           H,           O + 56 * 1,  R          },
3181         {J,           L,           R,           P          },
3182         {N,           J,           P,           S          },
3183         {H + 56 * 2,  N,           S,           O + 56 * 3 },
3184         {M,           G + 56 * 2,  O + 56 * 2,  T          },
3185         {II,          M,           T,           P          },
3186  /* layer 2 */
3187         {O,           Q,           Y,           U          },
3188         {Q,           P,           W,           Y          },
3189         {R,           O + 56 * 1,  U + 56 * 1,  Ap         },
3190         {P,           R,           Ap,          W          },
3191         {S,           P,           X,           Bp         },
3192         {O + 56 * 3,  S,           Bp,          V + 56 * 1 },
3193         {T,           O + 56 * 2,  V,           Z          },
3194         {P,           T,           Z,           X          },
3195  /* layer 3 */
3196         {U,           Y,           Ep,          Dp         },
3197         {Y,           W,           Cp,          Ep         },
3198         {Ap,          U + 56 * 1,  Dp + 56 * 1, Gp         },
3199         {W,           Ap,          Gp,          Cp         },
3200         {Bp,          X,           Cp + 56 * 2, Fp         },
3201         {V + 56 * 1,  Bp,          Fp,          Dp + 56 * 1},
3202         {Z,           V,           Dp,          Hp         },
3203         {X,           Z,           Hp,          Cp + 56 * 2},
3204  /* layer 4 */
3205         {Dp,          Ep,          Mp,          Kp         },
3206         {Ep,          Cp,          Ip,          Mp         },
3207         {Gp,          Dp + 56 * 1, Lp,          Np         },
3208         {Cp,          Gp,          Np,          Jp         },
3209         {Fp,          Cp + 56 * 2, Jp + 56 * 2, Pp         },
3210         {Dp + 56 * 1, Fp,          Pp,          Lp         },
3211         {Hp,          Dp,          Kp,          Op         },
3212         {Cp + 56 * 2, Hp,          Op,          Ip + 56 * 2},
3213  /* layer 5 */
3214         {Kp,          Mp,          Sp,          Rp         },
3215         {Mp,          Ip,          Qp,          Sp         },
3216         {Np,          Lp,          Rp,          Tp         },
3217         {Jp,          Np,          Tp,          Qp + 56 * 1},
3218         {Pp,          Jp + 56 * 2, Qp + 56 * 3, Up         },
3219         {Lp,          Pp,          Up,          Rp         },
3220         {Op,          Kp,          Rp,          Vp         },
3221         {Ip + 56 * 2, Op,          Vp,          Qp + 56 * 2},
3222  /* layer 6 */
3223         {Rp,          Sp,          Aq,          Yp         },
3224         {Sp,          Qp,          Wp,          Aq         },
3225         {Tp,          Rp,          Yp,          Cq         },
3226         {Qp + 56 * 1, Tp,          Cq,          Wp + 56 * 1},
3227         {Up,          Qp + 56 * 3, Xp + 56 * 1, Dq         },
3228         {Rp,          Up,          Dq,          Zp         },
3229         {Vp,          Rp,          Zp,          Bq         },
3230         {Qp + 56 * 2, Vp,          Bq,          Xp         },
3231  /* layer 7 (the top is the periodic image of the bottom of layer 0) */
3232         {Yp,          Aq,          C + 56 * 4,  A + 56 * 4 },
3233         {Aq,          Wp,          B + 56 * 4,  C + 56 * 4 },
3234         {Cq,          Yp,          A + 56 * 4,  D + 56 * 4 },
3235         {Wp + 56 * 1, Cq,          D + 56 * 4,  B + 56 * 5 },
3236         {Dq,          Xp + 56 * 1, B + 56 * 5,  E + 56 * 4 },
3237         {Zp,          Dq,          E + 56 * 4,  A + 56 * 6 },
3238         {Bq,          Zp,          A + 56 * 6,  F + 56 * 4 },
3239         {Xp,          Bq,          F + 56 * 4,  B + 56 * 4 }
3240       };
3241       const PetscReal gamma                = PetscAcosReal((PetscSqrtReal(3.) - 1.) / PetscSqrtReal(2.)) / PETSC_PI;
3242       const PetscReal patternCoords[56][3] = {
3243         {1.,        0.,        0.  }, /* A  */
3244         {0.,        1.,        0.  }, /* B  */
3245         {gamma,     gamma,     0.  }, /* C  */
3246         {1 + gamma, 1 - gamma, 0.  }, /* D  */
3247         {2 - gamma, 2 - gamma, 0.  }, /* E  */
3248         {1 - gamma, 1 + gamma, 0.  }, /* F  */
3249 
3250         {.5,        0,         .25 }, /* G  */
3251         {1.5,       0.,        .25 }, /* H  */
3252         {.5,        1.,        .25 }, /* II */
3253         {1.5,       1.,        .25 }, /* J  */
3254         {.25,       .5,        .25 }, /* K  */
3255         {1.25,      .5,        .25 }, /* L  */
3256         {.75,       1.5,       .25 }, /* M  */
3257         {1.75,      1.5,       .25 }, /* N  */
3258 
3259         {0.,        0.,        .5  }, /* O  */
3260         {1.,        1.,        .5  }, /* P  */
3261         {gamma,     1 - gamma, .5  }, /* Q  */
3262         {1 + gamma, gamma,     .5  }, /* R  */
3263         {2 - gamma, 1 + gamma, .5  }, /* S  */
3264         {1 - gamma, 2 - gamma, .5  }, /* T  */
3265 
3266         {0.,        .5,        .75 }, /* U  */
3267         {0.,        1.5,       .75 }, /* V  */
3268         {1.,        .5,        .75 }, /* W  */
3269         {1.,        1.5,       .75 }, /* X  */
3270         {.5,        .75,       .75 }, /* Y  */
3271         {.5,        1.75,      .75 }, /* Z  */
3272         {1.5,       .25,       .75 }, /* Ap */
3273         {1.5,       1.25,      .75 }, /* Bp */
3274 
3275         {1.,        0.,        1.  }, /* Cp */
3276         {0.,        1.,        1.  }, /* Dp */
3277         {1 - gamma, 1 - gamma, 1.  }, /* Ep */
3278         {1 + gamma, 1 + gamma, 1.  }, /* Fp */
3279         {2 - gamma, gamma,     1.  }, /* Gp */
3280         {gamma,     2 - gamma, 1.  }, /* Hp */
3281 
3282         {.5,        0.,        1.25}, /* Ip */
3283         {1.5,       0.,        1.25}, /* Jp */
3284         {.5,        1.,        1.25}, /* Kp */
3285         {1.5,       1.,        1.25}, /* Lp */
3286         {.75,       .5,        1.25}, /* Mp */
3287         {1.75,      .5,        1.25}, /* Np */
3288         {.25,       1.5,       1.25}, /* Op */
3289         {1.25,      1.5,       1.25}, /* Pp */
3290 
3291         {0.,        0.,        1.5 }, /* Qp */
3292         {1.,        1.,        1.5 }, /* Rp */
3293         {1 - gamma, gamma,     1.5 }, /* Sp */
3294         {2 - gamma, 1 - gamma, 1.5 }, /* Tp */
3295         {1 + gamma, 2 - gamma, 1.5 }, /* Up */
3296         {gamma,     1 + gamma, 1.5 }, /* Vp */
3297 
3298         {0.,        .5,        1.75}, /* Wp */
3299         {0.,        1.5,       1.75}, /* Xp */
3300         {1.,        .5,        1.75}, /* Yp */
3301         {1.,        1.5,       1.75}, /* Zp */
3302         {.5,        .25,       1.75}, /* Aq */
3303         {.5,        1.25,      1.75}, /* Bq */
3304         {1.5,       .75,       1.75}, /* Cq */
3305         {1.5,       1.75,      1.75}, /* Dq */
3306       };
3307       PetscInt(*cells)[64][4] = NULL;
3308       PetscBool *seen;
3309       PetscInt  *vertToTrueVert;
3310       PetscInt   count;
3311 
3312       for (PetscInt i = 0; i < 3; i++) extentPlus[i] = extent[i] + 1;
3313       numBlocks = 1;
3314       for (PetscInt i = 0; i < 3; i++) numBlocks *= extent[i];
3315       numBlocksPlus = 1;
3316       for (PetscInt i = 0; i < 3; i++) numBlocksPlus *= extentPlus[i];
3317       numFaces = numBlocks * facesPerBlock;
3318       PetscCall(PetscMalloc1(numBlocks, &cells));
3319       PetscCall(PetscCalloc1(numBlocksPlus * vertsPerBlock, &seen));
3320       for (PetscInt k = 0; k < extent[2]; k++) {
3321         for (PetscInt j = 0; j < extent[1]; j++) {
3322           for (PetscInt i = 0; i < extent[0]; i++) {
3323             for (PetscInt f = 0; f < facesPerBlock; f++) {
3324               for (PetscInt v = 0; v < 4; v++) {
3325                 PetscInt vertRaw     = pattern[f][v];
3326                 PetscInt blockidx    = vertRaw / 56;
3327                 PetscInt patternvert = vertRaw % 56;
3328                 PetscInt xplus       = (blockidx & 1);
3329                 PetscInt yplus       = (blockidx & 2) >> 1;
3330                 PetscInt zplus       = (blockidx & 4) >> 2;
3331                 PetscInt zcoord      = (periodic && periodic[2] == DM_BOUNDARY_PERIODIC) ? ((k + zplus) % extent[2]) : (k + zplus);
3332                 PetscInt ycoord      = (periodic && periodic[1] == DM_BOUNDARY_PERIODIC) ? ((j + yplus) % extent[1]) : (j + yplus);
3333                 PetscInt xcoord      = (periodic && periodic[0] == DM_BOUNDARY_PERIODIC) ? ((i + xplus) % extent[0]) : (i + xplus);
3334                 PetscInt vert        = ((zcoord * extentPlus[1] + ycoord) * extentPlus[0] + xcoord) * 56 + patternvert;
3335 
3336                 cells[(k * extent[1] + j) * extent[0] + i][f][v] = vert;
3337                 seen[vert]                                       = PETSC_TRUE;
3338               }
3339             }
3340           }
3341         }
3342       }
3343       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++)
3344         if (seen[i]) numVertices++;
3345       count = 0;
3346       PetscCall(PetscMalloc1(numBlocksPlus * vertsPerBlock, &vertToTrueVert));
3347       PetscCall(PetscMalloc1(numVertices * 3, &vtxCoords));
3348       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) vertToTrueVert[i] = -1;
3349       for (PetscInt k = 0; k < extentPlus[2]; k++) {
3350         for (PetscInt j = 0; j < extentPlus[1]; j++) {
3351           for (PetscInt i = 0; i < extentPlus[0]; i++) {
3352             for (PetscInt v = 0; v < vertsPerBlock; v++) {
3353               PetscInt vIdx = ((k * extentPlus[1] + j) * extentPlus[0] + i) * vertsPerBlock + v;
3354 
3355               if (seen[vIdx]) {
3356                 PetscInt thisVert;
3357 
3358                 vertToTrueVert[vIdx] = thisVert = count++;
3359 
3360                 for (PetscInt d = 0; d < 3; d++) vtxCoords[3 * thisVert + d] = patternCoords[v][d];
3361                 vtxCoords[3 * thisVert + 0] += i * 2;
3362                 vtxCoords[3 * thisVert + 1] += j * 2;
3363                 vtxCoords[3 * thisVert + 2] += k * 2;
3364               }
3365             }
3366           }
3367         }
3368       }
3369       for (PetscInt i = 0; i < numBlocks; i++) {
3370         for (PetscInt f = 0; f < facesPerBlock; f++) {
3371           for (PetscInt v = 0; v < 4; v++) cells[i][f][v] = vertToTrueVert[cells[i][f][v]];
3372         }
3373       }
3374       PetscCall(PetscFree(vertToTrueVert));
3375       PetscCall(PetscFree(seen));
3376       cells_flat = cells[0][0];
3377       numEdges   = 0;
3378       for (PetscInt i = 0; i < numFaces; i++) {
3379         for (PetscInt e = 0; e < 4; e++) {
3380           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3381           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3382 
3383           for (PetscInt d = 0; d < 3; d++) {
3384             if (!periodic || periodic[0] != DM_BOUNDARY_PERIODIC) {
3385               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) numEdges++;
3386               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) numEdges++;
3387             }
3388           }
3389         }
3390       }
3391       PetscCall(PetscMalloc1(numEdges, &edges));
3392       PetscCall(PetscMalloc1(numEdges, &edgeSets));
3393       for (PetscInt edge = 0, i = 0; i < numFaces; i++) {
3394         for (PetscInt e = 0; e < 4; e++) {
3395           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3396           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3397 
3398           for (PetscInt d = 0; d < 3; d++) {
3399             if (!periodic || periodic[d] != DM_BOUNDARY_PERIODIC) {
3400               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) {
3401                 edges[edge][0]   = ev[0];
3402                 edges[edge][1]   = ev[1];
3403                 edgeSets[edge++] = 2 * d;
3404               }
3405               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) {
3406                 edges[edge][0]   = ev[0];
3407                 edges[edge][1]   = ev[1];
3408                 edgeSets[edge++] = 2 * d + 1;
3409               }
3410             }
3411           }
3412         }
3413       }
3414     }
3415     evalFunc   = TPSEvaluate_Gyroid;
3416     normalFunc = TPSExtrudeNormalFunc_Gyroid;
3417     break;
3418   }
3419 
3420   PetscCall(DMSetDimension(dm, topoDim));
3421   if (rank == 0) PetscCall(DMPlexBuildFromCellList(dm, numFaces, numVertices, 4, cells_flat));
3422   else PetscCall(DMPlexBuildFromCellList(dm, 0, 0, 0, NULL));
3423   PetscCall(PetscFree(cells_flat));
3424   {
3425     DM idm;
3426     PetscCall(DMPlexInterpolate(dm, &idm));
3427     PetscCall(DMPlexReplace_Internal(dm, &idm));
3428   }
3429   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, vtxCoords));
3430   else PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, NULL));
3431   PetscCall(PetscFree(vtxCoords));
3432 
3433   PetscCall(DMCreateLabel(dm, "Face Sets"));
3434   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3435   for (PetscInt e = 0; e < numEdges; e++) {
3436     PetscInt        njoin;
3437     const PetscInt *join, verts[] = {numFaces + edges[e][0], numFaces + edges[e][1]};
3438     PetscCall(DMPlexGetJoin(dm, 2, verts, &njoin, &join));
3439     PetscCheck(njoin == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected unique join of vertices %" PetscInt_FMT " and %" PetscInt_FMT, edges[e][0], edges[e][1]);
3440     PetscCall(DMLabelSetValue(label, join[0], edgeSets[e]));
3441     PetscCall(DMPlexRestoreJoin(dm, 2, verts, &njoin, &join));
3442   }
3443   PetscCall(PetscFree(edges));
3444   PetscCall(PetscFree(edgeSets));
3445   if (tps_distribute) {
3446     DM               pdm = NULL;
3447     PetscPartitioner part;
3448 
3449     PetscCall(DMPlexGetPartitioner(dm, &part));
3450     PetscCall(PetscPartitionerSetFromOptions(part));
3451     PetscCall(DMPlexDistribute(dm, 0, NULL, &pdm));
3452     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
3453     // Do not auto-distribute again
3454     PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
3455   }
3456 
3457   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3458   for (PetscInt refine = 0; refine < refinements; refine++) {
3459     PetscInt     m;
3460     DM           dmf;
3461     Vec          X;
3462     PetscScalar *x;
3463     PetscCall(DMRefine(dm, MPI_COMM_NULL, &dmf));
3464     PetscCall(DMPlexReplace_Internal(dm, &dmf));
3465 
3466     PetscCall(DMGetCoordinatesLocal(dm, &X));
3467     PetscCall(VecGetLocalSize(X, &m));
3468     PetscCall(VecGetArray(X, &x));
3469     for (PetscInt i = 0; i < m; i += 3) PetscCall(TPSNearestPoint(evalFunc, &x[i]));
3470     PetscCall(VecRestoreArray(X, &x));
3471   }
3472 
3473   // Face Sets has already been propagated to new vertices during refinement; this propagates to the initial vertices.
3474   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3475   PetscCall(DMPlexLabelComplete(dm, label));
3476 
3477   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
3478 
3479   if (thickness > 0) {
3480     DM              edm, cdm, ecdm;
3481     DMPlexTransform tr;
3482     const char     *prefix;
3483     PetscOptions    options;
3484     // Code from DMPlexExtrude
3485     PetscCall(DMPlexTransformCreate(PetscObjectComm((PetscObject)dm), &tr));
3486     PetscCall(DMPlexTransformSetDM(tr, dm));
3487     PetscCall(DMPlexTransformSetType(tr, DMPLEXEXTRUDE));
3488     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
3489     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)tr, prefix));
3490     PetscCall(PetscObjectGetOptions((PetscObject)dm, &options));
3491     PetscCall(PetscObjectSetOptions((PetscObject)tr, options));
3492     PetscCall(DMPlexTransformExtrudeSetLayers(tr, layers));
3493     PetscCall(DMPlexTransformExtrudeSetThickness(tr, thickness));
3494     PetscCall(DMPlexTransformExtrudeSetTensor(tr, PETSC_FALSE));
3495     PetscCall(DMPlexTransformExtrudeSetSymmetric(tr, PETSC_TRUE));
3496     PetscCall(DMPlexTransformExtrudeSetNormalFunction(tr, normalFunc));
3497     PetscCall(DMPlexTransformSetFromOptions(tr));
3498     PetscCall(PetscObjectSetOptions((PetscObject)tr, NULL));
3499     PetscCall(DMPlexTransformSetUp(tr));
3500     PetscCall(PetscObjectViewFromOptions((PetscObject)tr, NULL, "-dm_plex_tps_transform_view"));
3501     PetscCall(DMPlexTransformApply(tr, dm, &edm));
3502     PetscCall(DMCopyDisc(dm, edm));
3503     PetscCall(DMGetCoordinateDM(dm, &cdm));
3504     PetscCall(DMGetCoordinateDM(edm, &ecdm));
3505     PetscCall(DMCopyDisc(cdm, ecdm));
3506     PetscCall(DMPlexTransformCreateDiscLabels(tr, edm));
3507     PetscCall(DMPlexTransformDestroy(&tr));
3508     if (edm) {
3509       ((DM_Plex *)edm->data)->printFEM    = ((DM_Plex *)dm->data)->printFEM;
3510       ((DM_Plex *)edm->data)->printL2     = ((DM_Plex *)dm->data)->printL2;
3511       ((DM_Plex *)edm->data)->printLocate = ((DM_Plex *)dm->data)->printLocate;
3512     }
3513     PetscCall(DMPlexReplace_Internal(dm, &edm));
3514   }
3515   PetscFunctionReturn(PETSC_SUCCESS);
3516 }
3517 
3518 /*@
3519   DMPlexCreateTPSMesh - Create a distributed, interpolated mesh of a triply-periodic surface
3520 
3521   Collective
3522 
3523   Input Parameters:
3524 + comm           - The communicator for the `DM` object
3525 . tpstype        - Type of triply-periodic surface
3526 . extent         - Array of length 3 containing number of periods in each direction
3527 . periodic       - array of length 3 with periodicity, or `NULL` for non-periodic
3528 . tps_distribute - Distribute 2D manifold mesh prior to refinement and extrusion (more scalable)
3529 . refinements    - Number of factor-of-2 refinements of 2D manifold mesh
3530 . layers         - Number of cell layers extruded in normal direction
3531 - thickness      - Thickness in normal direction
3532 
3533   Output Parameter:
3534 . dm - The `DM` object
3535 
3536   Level: beginner
3537 
3538   Notes:
3539   This meshes the surface of the Schwarz P or Gyroid surfaces.  Schwarz P is is the simplest member of the triply-periodic minimal surfaces.
3540   https://en.wikipedia.org/wiki/Schwarz_minimal_surface#Schwarz_P_(%22Primitive%22) and can be cut with "clean" boundaries.
3541   The Gyroid (https://en.wikipedia.org/wiki/Gyroid) is another triply-periodic minimal surface with applications in additive manufacturing; it is much more difficult to "cut" since there are no planes of symmetry.
3542   Our implementation creates a very coarse mesh of the surface and refines (by 4-way splitting) as many times as requested.
3543   On each refinement, all vertices are projected to their nearest point on the surface.
3544   This projection could readily be extended to related surfaces.
3545 
3546   The face (edge) sets for the Schwarz P surface are numbered 1(-x), 2(+x), 3(-y), 4(+y), 5(-z), 6(+z).
3547   When the mesh is refined, "Face Sets" contain the new vertices (created during refinement).  Use `DMPlexLabelComplete()` to propagate to coarse-level vertices.
3548 
3549   Developer Notes:
3550   The Gyroid mesh does not currently mark boundary sets.
3551 
3552   References:
3553 . * - Maskery et al, Insights into the mechanical properties of several triply periodic minimal surface lattice structures made by polymer additive manufacturing, 2017.
3554   https://doi.org/10.1016/j.polymer.2017.11.049
3555 
3556 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMSetType()`, `DMCreate()`
3557 @*/
3558 PetscErrorCode DMPlexCreateTPSMesh(MPI_Comm comm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness, DM *dm)
3559 {
3560   PetscFunctionBegin;
3561   PetscCall(DMCreate(comm, dm));
3562   PetscCall(DMSetType(*dm, DMPLEX));
3563   PetscCall(DMPlexCreateTPSMesh_Internal(*dm, tpstype, extent, periodic, tps_distribute, refinements, layers, thickness));
3564   PetscFunctionReturn(PETSC_SUCCESS);
3565 }
3566 
3567 /*@
3568   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
3569 
3570   Collective
3571 
3572   Input Parameters:
3573 + comm    - The communicator for the `DM` object
3574 . dim     - The dimension
3575 . simplex - Use simplices, or tensor product cells
3576 - R       - The radius
3577 
3578   Output Parameter:
3579 . dm - The `DM` object
3580 
3581   Level: beginner
3582 
3583 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBallMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3584 @*/
3585 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscReal R, DM *dm)
3586 {
3587   PetscFunctionBegin;
3588   PetscAssertPointer(dm, 5);
3589   PetscCall(DMCreate(comm, dm));
3590   PetscCall(DMSetType(*dm, DMPLEX));
3591   PetscCall(DMPlexCreateSphereMesh_Internal(*dm, dim, simplex, R));
3592   PetscFunctionReturn(PETSC_SUCCESS);
3593 }
3594 
3595 static PetscErrorCode DMPlexCreateBallMesh_Internal(DM dm, PetscInt dim, PetscReal R)
3596 {
3597   DM      sdm, vol;
3598   DMLabel bdlabel;
3599 
3600   PetscFunctionBegin;
3601   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
3602   PetscCall(DMSetType(sdm, DMPLEX));
3603   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sdm, "bd_"));
3604   PetscCall(DMPlexCreateSphereMesh_Internal(sdm, dim - 1, PETSC_TRUE, R));
3605   PetscCall(DMSetFromOptions(sdm));
3606   PetscCall(DMViewFromOptions(sdm, NULL, "-dm_view"));
3607   PetscCall(DMPlexGenerate(sdm, NULL, PETSC_TRUE, &vol));
3608   PetscCall(DMDestroy(&sdm));
3609   PetscCall(DMPlexReplace_Internal(dm, &vol));
3610   PetscCall(DMCreateLabel(dm, "marker"));
3611   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
3612   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
3613   PetscCall(DMPlexLabelComplete(dm, bdlabel));
3614   PetscFunctionReturn(PETSC_SUCCESS);
3615 }
3616 
3617 /*@
3618   DMPlexCreateBallMesh - Creates a simplex mesh on the d-dimensional ball, B^d.
3619 
3620   Collective
3621 
3622   Input Parameters:
3623 + comm - The communicator for the `DM` object
3624 . dim  - The dimension
3625 - R    - The radius
3626 
3627   Output Parameter:
3628 . dm - The `DM` object
3629 
3630   Options Database Key:
3631 . bd_dm_refine - This will refine the surface mesh preserving the sphere geometry
3632 
3633   Level: beginner
3634 
3635 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3636 @*/
3637 PetscErrorCode DMPlexCreateBallMesh(MPI_Comm comm, PetscInt dim, PetscReal R, DM *dm)
3638 {
3639   PetscFunctionBegin;
3640   PetscCall(DMCreate(comm, dm));
3641   PetscCall(DMSetType(*dm, DMPLEX));
3642   PetscCall(DMPlexCreateBallMesh_Internal(*dm, dim, R));
3643   PetscFunctionReturn(PETSC_SUCCESS);
3644 }
3645 
3646 static PetscErrorCode DMPlexCreateReferenceCell_Internal(DM rdm, DMPolytopeType ct)
3647 {
3648   PetscFunctionBegin;
3649   switch (ct) {
3650   case DM_POLYTOPE_POINT: {
3651     PetscInt    numPoints[1]        = {1};
3652     PetscInt    coneSize[1]         = {0};
3653     PetscInt    cones[1]            = {0};
3654     PetscInt    coneOrientations[1] = {0};
3655     PetscScalar vertexCoords[1]     = {0.0};
3656 
3657     PetscCall(DMSetDimension(rdm, 0));
3658     PetscCall(DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3659   } break;
3660   case DM_POLYTOPE_SEGMENT: {
3661     PetscInt    numPoints[2]        = {2, 1};
3662     PetscInt    coneSize[3]         = {2, 0, 0};
3663     PetscInt    cones[2]            = {1, 2};
3664     PetscInt    coneOrientations[2] = {0, 0};
3665     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3666 
3667     PetscCall(DMSetDimension(rdm, 1));
3668     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3669   } break;
3670   case DM_POLYTOPE_POINT_PRISM_TENSOR: {
3671     PetscInt    numPoints[2]        = {2, 1};
3672     PetscInt    coneSize[3]         = {2, 0, 0};
3673     PetscInt    cones[2]            = {1, 2};
3674     PetscInt    coneOrientations[2] = {0, 0};
3675     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3676 
3677     PetscCall(DMSetDimension(rdm, 1));
3678     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3679   } break;
3680   case DM_POLYTOPE_TRIANGLE: {
3681     PetscInt    numPoints[2]        = {3, 1};
3682     PetscInt    coneSize[4]         = {3, 0, 0, 0};
3683     PetscInt    cones[3]            = {1, 2, 3};
3684     PetscInt    coneOrientations[3] = {0, 0, 0};
3685     PetscScalar vertexCoords[6]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0};
3686 
3687     PetscCall(DMSetDimension(rdm, 2));
3688     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3689   } break;
3690   case DM_POLYTOPE_QUADRILATERAL: {
3691     PetscInt    numPoints[2]        = {4, 1};
3692     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3693     PetscInt    cones[4]            = {1, 2, 3, 4};
3694     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3695     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0};
3696 
3697     PetscCall(DMSetDimension(rdm, 2));
3698     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3699   } break;
3700   case DM_POLYTOPE_SEG_PRISM_TENSOR: {
3701     PetscInt    numPoints[2]        = {4, 1};
3702     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3703     PetscInt    cones[4]            = {1, 2, 3, 4};
3704     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3705     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0};
3706 
3707     PetscCall(DMSetDimension(rdm, 2));
3708     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3709   } break;
3710   case DM_POLYTOPE_TETRAHEDRON: {
3711     PetscInt    numPoints[2]        = {4, 1};
3712     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3713     PetscInt    cones[4]            = {1, 2, 3, 4};
3714     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3715     PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0};
3716 
3717     PetscCall(DMSetDimension(rdm, 3));
3718     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3719   } break;
3720   case DM_POLYTOPE_HEXAHEDRON: {
3721     PetscInt    numPoints[2]        = {8, 1};
3722     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3723     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3724     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3725     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3726 
3727     PetscCall(DMSetDimension(rdm, 3));
3728     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3729   } break;
3730   case DM_POLYTOPE_TRI_PRISM: {
3731     PetscInt    numPoints[2]        = {6, 1};
3732     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3733     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3734     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3735     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3736 
3737     PetscCall(DMSetDimension(rdm, 3));
3738     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3739   } break;
3740   case DM_POLYTOPE_TRI_PRISM_TENSOR: {
3741     PetscInt    numPoints[2]        = {6, 1};
3742     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3743     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3744     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3745     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3746 
3747     PetscCall(DMSetDimension(rdm, 3));
3748     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3749   } break;
3750   case DM_POLYTOPE_QUAD_PRISM_TENSOR: {
3751     PetscInt    numPoints[2]        = {8, 1};
3752     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3753     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3754     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3755     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3756 
3757     PetscCall(DMSetDimension(rdm, 3));
3758     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3759   } break;
3760   case DM_POLYTOPE_PYRAMID: {
3761     PetscInt    numPoints[2]        = {5, 1};
3762     PetscInt    coneSize[6]         = {5, 0, 0, 0, 0, 0};
3763     PetscInt    cones[5]            = {1, 2, 3, 4, 5};
3764     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3765     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, 0.0, 0.0, 1.0};
3766 
3767     PetscCall(DMSetDimension(rdm, 3));
3768     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3769   } break;
3770   default:
3771     SETERRQ(PetscObjectComm((PetscObject)rdm), PETSC_ERR_ARG_WRONG, "Cannot create reference cell for cell type %s", DMPolytopeTypes[ct]);
3772   }
3773   {
3774     PetscInt Nv, v;
3775 
3776     /* Must create the celltype label here so that we do not automatically try to compute the types */
3777     PetscCall(DMCreateLabel(rdm, "celltype"));
3778     PetscCall(DMPlexSetCellType(rdm, 0, ct));
3779     PetscCall(DMPlexGetChart(rdm, NULL, &Nv));
3780     for (v = 1; v < Nv; ++v) PetscCall(DMPlexSetCellType(rdm, v, DM_POLYTOPE_POINT));
3781   }
3782   PetscCall(DMPlexInterpolateInPlace_Internal(rdm));
3783   PetscCall(PetscObjectSetName((PetscObject)rdm, DMPolytopeTypes[ct]));
3784   PetscFunctionReturn(PETSC_SUCCESS);
3785 }
3786 
3787 /*@
3788   DMPlexCreateReferenceCell - Create a `DMPLEX` with the appropriate FEM reference cell
3789 
3790   Collective
3791 
3792   Input Parameters:
3793 + comm - The communicator
3794 - ct   - The cell type of the reference cell
3795 
3796   Output Parameter:
3797 . refdm - The reference cell
3798 
3799   Level: intermediate
3800 
3801 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBoxMesh()`
3802 @*/
3803 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, DMPolytopeType ct, DM *refdm)
3804 {
3805   PetscFunctionBegin;
3806   PetscCall(DMCreate(comm, refdm));
3807   PetscCall(DMSetType(*refdm, DMPLEX));
3808   PetscCall(DMPlexCreateReferenceCell_Internal(*refdm, ct));
3809   PetscFunctionReturn(PETSC_SUCCESS);
3810 }
3811 
3812 static PetscErrorCode DMPlexCreateBoundaryLabel_Private(DM dm, const char name[])
3813 {
3814   DM        plex;
3815   DMLabel   label;
3816   PetscBool hasLabel;
3817 
3818   PetscFunctionBegin;
3819   PetscCall(DMHasLabel(dm, name, &hasLabel));
3820   if (hasLabel) PetscFunctionReturn(PETSC_SUCCESS);
3821   PetscCall(DMCreateLabel(dm, name));
3822   PetscCall(DMGetLabel(dm, name, &label));
3823   PetscCall(DMConvert(dm, DMPLEX, &plex));
3824   PetscCall(DMPlexMarkBoundaryFaces(plex, 1, label));
3825   PetscCall(DMPlexLabelComplete(plex, label));
3826   PetscCall(DMDestroy(&plex));
3827   PetscFunctionReturn(PETSC_SUCCESS);
3828 }
3829 
3830 /*
3831   We use the last coordinate as the radius, the inner radius is lower[dim-1] and the outer radius is upper[dim-1]. Then we map the first coordinate around the circle.
3832 
3833     (x, y) -> (r, theta) = (x[1], (x[0] - lower[0]) * 2\pi/(upper[0] - lower[0]))
3834 */
3835 static void boxToAnnulus(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
3836 {
3837   const PetscReal low = PetscRealPart(constants[0]);
3838   const PetscReal upp = PetscRealPart(constants[1]);
3839   const PetscReal r   = PetscRealPart(u[1]);
3840   const PetscReal th  = 2. * PETSC_PI * (PetscRealPart(u[0]) - low) / (upp - low);
3841 
3842   f0[0] = r * PetscCosReal(th);
3843   f0[1] = r * PetscSinReal(th);
3844 }
3845 
3846 const char *const DMPlexShapes[] = {"box", "box_surface", "ball", "sphere", "cylinder", "schwarz_p", "gyroid", "doublet", "annulus", "hypercubic", "zbox", "unknown", "DMPlexShape", "DM_SHAPE_", NULL};
3847 
3848 static PetscErrorCode DMPlexCreateFromOptions_Internal(PetscOptionItems *PetscOptionsObject, PetscBool *useCoordSpace, DM dm)
3849 {
3850   DMPlexShape    shape   = DM_SHAPE_BOX;
3851   DMPolytopeType cell    = DM_POLYTOPE_TRIANGLE;
3852   PetscInt       dim     = 2;
3853   PetscBool      simplex = PETSC_TRUE, interpolate = PETSC_TRUE, adjCone = PETSC_FALSE, adjClosure = PETSC_TRUE, refDomain = PETSC_FALSE;
3854   PetscBool      flg, flg2, fflg, bdfflg, nameflg;
3855   MPI_Comm       comm;
3856   char           filename[PETSC_MAX_PATH_LEN]   = "<unspecified>";
3857   char           bdFilename[PETSC_MAX_PATH_LEN] = "<unspecified>";
3858   char           plexname[PETSC_MAX_PATH_LEN]   = "";
3859 
3860   PetscFunctionBegin;
3861   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromOptions, dm, 0, 0, 0));
3862   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3863   /* TODO Turn this into a registration interface */
3864   PetscCall(PetscOptionsString("-dm_plex_filename", "File containing a mesh", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &fflg));
3865   PetscCall(PetscOptionsString("-dm_plex_boundary_filename", "File containing a mesh boundary", "DMPlexCreateFromFile", bdFilename, bdFilename, sizeof(bdFilename), &bdfflg));
3866   PetscCall(PetscOptionsString("-dm_plex_name", "Name of the mesh in the file", "DMPlexCreateFromFile", plexname, plexname, sizeof(plexname), &nameflg));
3867   PetscCall(PetscOptionsEnum("-dm_plex_cell", "Cell shape", "", DMPolytopeTypes, (PetscEnum)cell, (PetscEnum *)&cell, NULL));
3868   PetscCall(PetscOptionsBool("-dm_plex_reference_cell_domain", "Use a reference cell domain", "", refDomain, &refDomain, NULL));
3869   PetscCall(PetscOptionsEnum("-dm_plex_shape", "Shape for built-in mesh", "", DMPlexShapes, (PetscEnum)shape, (PetscEnum *)&shape, &flg));
3870   PetscCall(PetscOptionsBoundedInt("-dm_plex_dim", "Topological dimension of the mesh", "DMGetDimension", dim, &dim, &flg, 0));
3871   PetscCheck(dim >= 0, comm, PETSC_ERR_ARG_OUTOFRANGE, "Dimension %" PetscInt_FMT " should be in [0, infinity)", dim);
3872   PetscCall(PetscOptionsBool("-dm_plex_simplex", "Mesh cell shape", "", simplex, &simplex, &flg));
3873   PetscCall(PetscOptionsBool("-dm_plex_interpolate", "Flag to create edges and faces automatically", "", interpolate, &interpolate, &flg));
3874   PetscCall(PetscOptionsBool("-dm_plex_adj_cone", "Set adjacency direction", "DMSetBasicAdjacency", adjCone, &adjCone, &flg));
3875   PetscCall(PetscOptionsBool("-dm_plex_adj_closure", "Set adjacency size", "DMSetBasicAdjacency", adjClosure, &adjClosure, &flg2));
3876   if (flg || flg2) PetscCall(DMSetBasicAdjacency(dm, adjCone, adjClosure));
3877 
3878   switch (cell) {
3879   case DM_POLYTOPE_POINT:
3880   case DM_POLYTOPE_SEGMENT:
3881   case DM_POLYTOPE_POINT_PRISM_TENSOR:
3882   case DM_POLYTOPE_TRIANGLE:
3883   case DM_POLYTOPE_QUADRILATERAL:
3884   case DM_POLYTOPE_TETRAHEDRON:
3885   case DM_POLYTOPE_HEXAHEDRON:
3886     *useCoordSpace = PETSC_TRUE;
3887     break;
3888   default:
3889     *useCoordSpace = PETSC_FALSE;
3890     break;
3891   }
3892 
3893   if (fflg) {
3894     DM dmnew;
3895 
3896     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), filename, plexname, interpolate, &dmnew));
3897     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3898     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3899   } else if (refDomain) {
3900     PetscCall(DMPlexCreateReferenceCell_Internal(dm, cell));
3901   } else if (bdfflg) {
3902     DM bdm, dmnew;
3903 
3904     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), bdFilename, plexname, interpolate, &bdm));
3905     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)bdm, "bd_"));
3906     PetscCall(DMSetFromOptions(bdm));
3907     PetscCall(DMPlexGenerate(bdm, NULL, interpolate, &dmnew));
3908     PetscCall(DMDestroy(&bdm));
3909     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3910     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3911   } else {
3912     PetscCall(PetscObjectSetName((PetscObject)dm, DMPlexShapes[shape]));
3913     switch (shape) {
3914     case DM_SHAPE_BOX:
3915     case DM_SHAPE_ZBOX:
3916     case DM_SHAPE_ANNULUS: {
3917       PetscInt       faces[3]  = {0, 0, 0};
3918       PetscReal      lower[3]  = {0, 0, 0};
3919       PetscReal      upper[3]  = {1, 1, 1};
3920       DMBoundaryType bdt[3]    = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3921       PetscBool      isAnnular = shape == DM_SHAPE_ANNULUS ? PETSC_TRUE : PETSC_FALSE;
3922       PetscInt       i, n;
3923 
3924       n = dim;
3925       for (i = 0; i < dim; ++i) faces[i] = (dim == 1 ? 1 : 4 - dim);
3926       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3927       n = 3;
3928       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3929       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3930       n = 3;
3931       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3932       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3933       n = 3;
3934       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
3935       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3936 
3937       PetscCheck(!isAnnular || dim == 2, comm, PETSC_ERR_ARG_OUTOFRANGE, "Only two dimensional annuli have been implemented");
3938       if (isAnnular)
3939         for (i = 0; i < dim - 1; ++i) bdt[i] = DM_BOUNDARY_PERIODIC;
3940 
3941       switch (cell) {
3942       case DM_POLYTOPE_TRI_PRISM_TENSOR:
3943         PetscCall(DMPlexCreateWedgeBoxMesh_Internal(dm, faces, lower, upper, bdt));
3944         if (!interpolate) {
3945           DM udm;
3946 
3947           PetscCall(DMPlexUninterpolate(dm, &udm));
3948           PetscCall(DMPlexReplace_Internal(dm, &udm));
3949         }
3950         break;
3951       default:
3952         PetscCall(DMPlexCreateBoxMesh_Internal(dm, shape, dim, simplex, faces, lower, upper, bdt, interpolate));
3953         break;
3954       }
3955       if (isAnnular) {
3956         DM          cdm;
3957         PetscDS     cds;
3958         PetscScalar bounds[2] = {lower[0], upper[0]};
3959 
3960         // Fix coordinates for annular region
3961         PetscCall(DMSetPeriodicity(dm, NULL, NULL, NULL));
3962         PetscCall(DMSetCellCoordinatesLocal(dm, NULL));
3963         PetscCall(DMSetCellCoordinates(dm, NULL));
3964         PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_TRUE, NULL));
3965         PetscCall(DMGetCoordinateDM(dm, &cdm));
3966         PetscCall(DMGetDS(cdm, &cds));
3967         PetscCall(PetscDSSetConstants(cds, 2, bounds));
3968         PetscCall(DMPlexRemapGeometry(dm, 0.0, boxToAnnulus));
3969       }
3970     } break;
3971     case DM_SHAPE_BOX_SURFACE: {
3972       PetscInt  faces[3] = {0, 0, 0};
3973       PetscReal lower[3] = {0, 0, 0};
3974       PetscReal upper[3] = {1, 1, 1};
3975       PetscInt  i, n;
3976 
3977       n = dim + 1;
3978       for (i = 0; i < dim + 1; ++i) faces[i] = (dim + 1 == 1 ? 1 : 4 - (dim + 1));
3979       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3980       n = 3;
3981       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3982       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3983       n = 3;
3984       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3985       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3986       PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(dm, dim + 1, faces, lower, upper, interpolate));
3987     } break;
3988     case DM_SHAPE_SPHERE: {
3989       PetscReal R = 1.0;
3990 
3991       PetscCall(PetscOptionsReal("-dm_plex_sphere_radius", "Radius of the sphere", "", R, &R, &flg));
3992       PetscCall(DMPlexCreateSphereMesh_Internal(dm, dim, simplex, R));
3993     } break;
3994     case DM_SHAPE_BALL: {
3995       PetscReal R = 1.0;
3996 
3997       PetscCall(PetscOptionsReal("-dm_plex_ball_radius", "Radius of the ball", "", R, &R, &flg));
3998       PetscCall(DMPlexCreateBallMesh_Internal(dm, dim, R));
3999     } break;
4000     case DM_SHAPE_CYLINDER: {
4001       DMBoundaryType bdt = DM_BOUNDARY_NONE;
4002       PetscInt       Nw  = 6;
4003 
4004       PetscCall(PetscOptionsEnum("-dm_plex_cylinder_bd", "Boundary type in the z direction", "", DMBoundaryTypes, (PetscEnum)bdt, (PetscEnum *)&bdt, NULL));
4005       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_wedges", "Number of wedges around the cylinder", "", Nw, &Nw, NULL));
4006       switch (cell) {
4007       case DM_POLYTOPE_TRI_PRISM_TENSOR:
4008         PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(dm, Nw, interpolate));
4009         break;
4010       default:
4011         PetscCall(DMPlexCreateHexCylinderMesh_Internal(dm, bdt));
4012         break;
4013       }
4014     } break;
4015     case DM_SHAPE_SCHWARZ_P: // fallthrough
4016     case DM_SHAPE_GYROID: {
4017       PetscInt       extent[3] = {1, 1, 1}, refine = 0, layers = 0, three;
4018       PetscReal      thickness   = 0.;
4019       DMBoundaryType periodic[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
4020       DMPlexTPSType  tps_type    = shape == DM_SHAPE_SCHWARZ_P ? DMPLEX_TPS_SCHWARZ_P : DMPLEX_TPS_GYROID;
4021       PetscBool      tps_distribute;
4022       PetscCall(PetscOptionsIntArray("-dm_plex_tps_extent", "Number of replicas for each of three dimensions", NULL, extent, (three = 3, &three), NULL));
4023       PetscCall(PetscOptionsInt("-dm_plex_tps_refine", "Number of refinements", NULL, refine, &refine, NULL));
4024       PetscCall(PetscOptionsEnumArray("-dm_plex_tps_periodic", "Periodicity in each of three dimensions", NULL, DMBoundaryTypes, (PetscEnum *)periodic, (three = 3, &three), NULL));
4025       PetscCall(PetscOptionsInt("-dm_plex_tps_layers", "Number of layers in volumetric extrusion (or zero to not extrude)", NULL, layers, &layers, NULL));
4026       PetscCall(PetscOptionsReal("-dm_plex_tps_thickness", "Thickness of volumetric extrusion", NULL, thickness, &thickness, NULL));
4027       PetscCall(DMPlexDistributeGetDefault(dm, &tps_distribute));
4028       PetscCall(PetscOptionsBool("-dm_plex_tps_distribute", "Distribute the 2D mesh prior to refinement and extrusion", NULL, tps_distribute, &tps_distribute, NULL));
4029       PetscCall(DMPlexCreateTPSMesh_Internal(dm, tps_type, extent, periodic, tps_distribute, refine, layers, thickness));
4030     } break;
4031     case DM_SHAPE_DOUBLET: {
4032       DM        dmnew;
4033       PetscReal rl = 0.0;
4034 
4035       PetscCall(PetscOptionsReal("-dm_plex_doublet_refinementlimit", "Refinement limit", NULL, rl, &rl, NULL));
4036       PetscCall(DMPlexCreateDoublet(PetscObjectComm((PetscObject)dm), dim, simplex, interpolate, rl, &dmnew));
4037       PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4038       PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4039     } break;
4040     case DM_SHAPE_HYPERCUBIC: {
4041       PetscInt       *edges;
4042       PetscReal      *lower, *upper;
4043       DMBoundaryType *bdt;
4044       PetscInt        n, d;
4045 
4046       *useCoordSpace = PETSC_FALSE;
4047       PetscCall(PetscMalloc4(dim, &edges, dim, &lower, dim, &upper, dim, &bdt));
4048       for (d = 0; d < dim; ++d) {
4049         edges[d] = 1;
4050         lower[d] = 0.;
4051         upper[d] = 1.;
4052         bdt[d]   = DM_BOUNDARY_PERIODIC;
4053       }
4054       n = dim;
4055       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", edges, &n, &flg));
4056       n = dim;
4057       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
4058       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4059       n = dim;
4060       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
4061       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4062       n = dim;
4063       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
4064       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4065       PetscCall(DMPlexCreateHypercubicMesh_Internal(dm, dim, lower, upper, edges, bdt));
4066       PetscCall(PetscFree4(edges, lower, upper, bdt));
4067     } break;
4068     default:
4069       SETERRQ(comm, PETSC_ERR_SUP, "Domain shape %s is unsupported", DMPlexShapes[shape]);
4070     }
4071   }
4072   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
4073   if (!((PetscObject)dm)->name && nameflg) PetscCall(PetscObjectSetName((PetscObject)dm, plexname));
4074   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromOptions, dm, 0, 0, 0));
4075   PetscFunctionReturn(PETSC_SUCCESS);
4076 }
4077 
4078 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
4079 {
4080   DM_Plex  *mesh = (DM_Plex *)dm->data;
4081   PetscBool flg, flg2;
4082   char      bdLabel[PETSC_MAX_PATH_LEN];
4083 
4084   PetscFunctionBegin;
4085   /* Handle viewing */
4086   PetscCall(PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL));
4087   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fem", "Debug output level for all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL, 0));
4088   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fvm", "Debug output level for all fvm computations", "DMPlexSNESComputeResidualFVM", 0, &mesh->printFVM, NULL, 0));
4089   PetscCall(PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL));
4090   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL, 0));
4091   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_locate", "Debug output level all point location computations", "DMLocatePoints", 0, &mesh->printLocate, NULL, 0));
4092   PetscCall(DMMonitorSetFromOptions(dm, "-dm_plex_monitor_throughput", "Monitor the simulation throughput", "DMPlexMonitorThroughput", DMPlexMonitorThroughput, NULL, &flg));
4093   if (flg) PetscCall(PetscLogDefaultBegin());
4094   /* Labeling */
4095   PetscCall(PetscOptionsString("-dm_plex_boundary_label", "Label to mark the mesh boundary", "", bdLabel, bdLabel, sizeof(bdLabel), &flg));
4096   if (flg) PetscCall(DMPlexCreateBoundaryLabel_Private(dm, bdLabel));
4097   /* Point Location */
4098   PetscCall(PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL));
4099   /* Partitioning and distribution */
4100   PetscCall(PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL));
4101   /* Generation and remeshing */
4102   PetscCall(PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL));
4103   /* Projection behavior */
4104   PetscCall(PetscOptionsBoundedInt("-dm_plex_max_projection_height", "Maximum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL, 0));
4105   PetscCall(PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL));
4106   /* Checking structure */
4107   {
4108     PetscBool all = PETSC_FALSE;
4109 
4110     PetscCall(PetscOptionsBool("-dm_plex_check_all", "Perform all basic checks", "DMPlexCheck", PETSC_FALSE, &all, NULL));
4111     if (all) {
4112       PetscCall(DMPlexCheck(dm));
4113     } else {
4114       PetscCall(PetscOptionsBool("-dm_plex_check_symmetry", "Check that the adjacency information in the mesh is symmetric", "DMPlexCheckSymmetry", PETSC_FALSE, &flg, &flg2));
4115       if (flg && flg2) PetscCall(DMPlexCheckSymmetry(dm));
4116       PetscCall(PetscOptionsBool("-dm_plex_check_skeleton", "Check that each cell has the correct number of vertices (only for homogeneous simplex or tensor meshes)", "DMPlexCheckSkeleton", PETSC_FALSE, &flg, &flg2));
4117       if (flg && flg2) PetscCall(DMPlexCheckSkeleton(dm, 0));
4118       PetscCall(PetscOptionsBool("-dm_plex_check_faces", "Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type", "DMPlexCheckFaces", PETSC_FALSE, &flg, &flg2));
4119       if (flg && flg2) PetscCall(DMPlexCheckFaces(dm, 0));
4120       PetscCall(PetscOptionsBool("-dm_plex_check_geometry", "Check that cells have positive volume", "DMPlexCheckGeometry", PETSC_FALSE, &flg, &flg2));
4121       if (flg && flg2) PetscCall(DMPlexCheckGeometry(dm));
4122       PetscCall(PetscOptionsBool("-dm_plex_check_pointsf", "Check some necessary conditions for PointSF", "DMPlexCheckPointSF", PETSC_FALSE, &flg, &flg2));
4123       if (flg && flg2) PetscCall(DMPlexCheckPointSF(dm, NULL, PETSC_FALSE));
4124       PetscCall(PetscOptionsBool("-dm_plex_check_interface_cones", "Check points on inter-partition interfaces have conforming order of cone points", "DMPlexCheckInterfaceCones", PETSC_FALSE, &flg, &flg2));
4125       if (flg && flg2) PetscCall(DMPlexCheckInterfaceCones(dm));
4126     }
4127     PetscCall(PetscOptionsBool("-dm_plex_check_cell_shape", "Check cell shape", "DMPlexCheckCellShape", PETSC_FALSE, &flg, &flg2));
4128     if (flg && flg2) PetscCall(DMPlexCheckCellShape(dm, PETSC_TRUE, PETSC_DETERMINE));
4129   }
4130   {
4131     PetscReal scale = 1.0;
4132 
4133     PetscCall(PetscOptionsReal("-dm_plex_scale", "Scale factor for mesh coordinates", "DMPlexScale", scale, &scale, &flg));
4134     if (flg) {
4135       Vec coordinates, coordinatesLocal;
4136 
4137       PetscCall(DMGetCoordinates(dm, &coordinates));
4138       PetscCall(DMGetCoordinatesLocal(dm, &coordinatesLocal));
4139       PetscCall(VecScale(coordinates, scale));
4140       PetscCall(VecScale(coordinatesLocal, scale));
4141     }
4142   }
4143   PetscCall(PetscPartitionerSetFromOptions(mesh->partitioner));
4144   PetscFunctionReturn(PETSC_SUCCESS);
4145 }
4146 
4147 PetscErrorCode DMSetFromOptions_Overlap_Plex(DM dm, PetscOptionItems *PetscOptionsObject, PetscInt *overlap)
4148 {
4149   PetscInt  numOvLabels = 16, numOvExLabels = 16;
4150   char     *ovLabelNames[16], *ovExLabelNames[16];
4151   PetscInt  numOvValues = 16, numOvExValues = 16, l;
4152   PetscBool flg;
4153 
4154   PetscFunctionBegin;
4155   PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", *overlap, overlap, NULL, 0));
4156   PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_labels", "List of overlap label names", "DMPlexDistribute", ovLabelNames, &numOvLabels, &flg));
4157   if (!flg) numOvLabels = 0;
4158   if (numOvLabels) {
4159     ((DM_Plex *)dm->data)->numOvLabels = numOvLabels;
4160     for (l = 0; l < numOvLabels; ++l) {
4161       PetscCall(DMGetLabel(dm, ovLabelNames[l], &((DM_Plex *)dm->data)->ovLabels[l]));
4162       PetscCheck(((DM_Plex *)dm->data)->ovLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovLabelNames[l]);
4163       PetscCall(PetscFree(ovLabelNames[l]));
4164     }
4165     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_values", "List of overlap label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovValues, &numOvValues, &flg));
4166     if (!flg) numOvValues = 0;
4167     PetscCheck(numOvLabels == numOvValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvLabels, numOvValues);
4168 
4169     PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_exclude_labels", "List of overlap exclude label names", "DMPlexDistribute", ovExLabelNames, &numOvExLabels, &flg));
4170     if (!flg) numOvExLabels = 0;
4171     ((DM_Plex *)dm->data)->numOvExLabels = numOvExLabels;
4172     for (l = 0; l < numOvExLabels; ++l) {
4173       PetscCall(DMGetLabel(dm, ovExLabelNames[l], &((DM_Plex *)dm->data)->ovExLabels[l]));
4174       PetscCheck(((DM_Plex *)dm->data)->ovExLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovExLabelNames[l]);
4175       PetscCall(PetscFree(ovExLabelNames[l]));
4176     }
4177     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_exclude_values", "List of overlap exclude label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovExValues, &numOvExValues, &flg));
4178     if (!flg) numOvExValues = 0;
4179     PetscCheck(numOvExLabels == numOvExValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of exclude labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvExLabels, numOvExValues);
4180   }
4181   PetscFunctionReturn(PETSC_SUCCESS);
4182 }
4183 
4184 static PetscErrorCode DMSetFromOptions_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
4185 {
4186   PetscFunctionList        ordlist;
4187   char                     oname[256];
4188   DMPlexReorderDefaultFlag reorder;
4189   PetscReal                volume    = -1.0;
4190   PetscInt                 prerefine = 0, refine = 0, r, coarsen = 0, overlap = 0, extLayers = 0, dim;
4191   PetscBool uniformOrig = PETSC_FALSE, created = PETSC_FALSE, uniform = PETSC_TRUE, distribute, saveSF = PETSC_FALSE, interpolate = PETSC_TRUE, coordSpace = PETSC_TRUE, remap = PETSC_TRUE, ghostCells = PETSC_FALSE, isHierarchy, ignoreModel = PETSC_FALSE, flg;
4192 
4193   PetscFunctionBegin;
4194   PetscOptionsHeadBegin(PetscOptionsObject, "DMPlex Options");
4195   if (dm->cloneOpts) goto non_refine;
4196   /* Handle automatic creation */
4197   PetscCall(DMGetDimension(dm, &dim));
4198   if (dim < 0) {
4199     PetscCall(DMPlexCreateFromOptions_Internal(PetscOptionsObject, &coordSpace, dm));
4200     created = PETSC_TRUE;
4201   }
4202   PetscCall(DMGetDimension(dm, &dim));
4203   /* Handle interpolation before distribution */
4204   PetscCall(PetscOptionsBool("-dm_plex_interpolate_pre", "Flag to interpolate mesh before distribution", "", interpolate, &interpolate, &flg));
4205   if (flg) {
4206     DMPlexInterpolatedFlag interpolated;
4207 
4208     PetscCall(DMPlexIsInterpolated(dm, &interpolated));
4209     if (interpolated == DMPLEX_INTERPOLATED_FULL && !interpolate) {
4210       DM udm;
4211 
4212       PetscCall(DMPlexUninterpolate(dm, &udm));
4213       PetscCall(DMPlexReplace_Internal(dm, &udm));
4214     } else if (interpolated != DMPLEX_INTERPOLATED_FULL && interpolate) {
4215       DM idm;
4216 
4217       PetscCall(DMPlexInterpolate(dm, &idm));
4218       PetscCall(DMPlexReplace_Internal(dm, &idm));
4219     }
4220   }
4221   /* Handle DMPlex refinement before distribution */
4222   PetscCall(PetscOptionsBool("-dm_refine_ignore_model", "Flag to ignore the geometry model when refining", "DMCreate", ignoreModel, &ignoreModel, &flg));
4223   if (flg) ((DM_Plex *)dm->data)->ignoreModel = ignoreModel;
4224   PetscCall(DMPlexGetRefinementUniform(dm, &uniformOrig));
4225   PetscCall(PetscOptionsBoundedInt("-dm_refine_pre", "The number of refinements before distribution", "DMCreate", prerefine, &prerefine, NULL, 0));
4226   PetscCall(PetscOptionsBool("-dm_refine_remap_pre", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
4227   PetscCall(PetscOptionsBool("-dm_refine_uniform_pre", "Flag for uniform refinement before distribution", "DMCreate", uniform, &uniform, &flg));
4228   if (flg) PetscCall(DMPlexSetRefinementUniform(dm, uniform));
4229   PetscCall(PetscOptionsReal("-dm_refine_volume_limit_pre", "The maximum cell volume after refinement before distribution", "DMCreate", volume, &volume, &flg));
4230   if (flg) {
4231     PetscCall(DMPlexSetRefinementUniform(dm, PETSC_FALSE));
4232     PetscCall(DMPlexSetRefinementLimit(dm, volume));
4233     prerefine = PetscMax(prerefine, 1);
4234   }
4235   for (r = 0; r < prerefine; ++r) {
4236     DM             rdm;
4237     PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4238 
4239     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4240     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
4241     PetscCall(DMPlexReplace_Internal(dm, &rdm));
4242     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4243     if (coordFunc && remap) {
4244       PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4245       ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4246     }
4247   }
4248   PetscCall(DMPlexSetRefinementUniform(dm, uniformOrig));
4249   /* Handle DMPlex extrusion before distribution */
4250   PetscCall(PetscOptionsBoundedInt("-dm_extrude", "The number of layers to extrude", "", extLayers, &extLayers, NULL, 0));
4251   if (extLayers) {
4252     DM edm;
4253 
4254     PetscCall(DMExtrude(dm, extLayers, &edm));
4255     PetscCall(DMPlexReplace_Internal(dm, &edm));
4256     ((DM_Plex *)dm->data)->coordFunc = NULL;
4257     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4258     extLayers = 0;
4259     PetscCall(DMGetDimension(dm, &dim));
4260   }
4261   /* Handle DMPlex reordering before distribution */
4262   PetscCall(DMPlexReorderGetDefault(dm, &reorder));
4263   PetscCall(MatGetOrderingList(&ordlist));
4264   PetscCall(PetscStrncpy(oname, MATORDERINGNATURAL, sizeof(oname)));
4265   PetscCall(PetscOptionsFList("-dm_plex_reorder", "Set mesh reordering type", "DMPlexGetOrdering", ordlist, MATORDERINGNATURAL, oname, sizeof(oname), &flg));
4266   if (reorder == DMPLEX_REORDER_DEFAULT_TRUE || flg) {
4267     DM pdm;
4268     IS perm;
4269 
4270     PetscCall(DMPlexGetOrdering(dm, oname, NULL, &perm));
4271     PetscCall(DMPlexPermute(dm, perm, &pdm));
4272     PetscCall(ISDestroy(&perm));
4273     PetscCall(DMPlexReplace_Internal(dm, &pdm));
4274     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4275   }
4276   /* Handle DMPlex distribution */
4277   PetscCall(DMPlexDistributeGetDefault(dm, &distribute));
4278   PetscCall(PetscOptionsBool("-dm_distribute", "Flag to redistribute a mesh among processes", "DMPlexDistribute", distribute, &distribute, NULL));
4279   PetscCall(PetscOptionsBool("-dm_distribute_save_sf", "Flag to save the migration SF", "DMPlexSetMigrationSF", saveSF, &saveSF, NULL));
4280   PetscCall(DMSetFromOptions_Overlap_Plex(dm, PetscOptionsObject, &overlap));
4281   if (distribute) {
4282     DM               pdm = NULL;
4283     PetscPartitioner part;
4284     PetscSF          sfMigration;
4285 
4286     PetscCall(DMPlexGetPartitioner(dm, &part));
4287     PetscCall(PetscPartitionerSetFromOptions(part));
4288     PetscCall(DMPlexDistribute(dm, overlap, &sfMigration, &pdm));
4289     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
4290     if (saveSF) PetscCall(DMPlexSetMigrationSF(dm, sfMigration));
4291     PetscCall(PetscSFDestroy(&sfMigration));
4292   }
4293   /* Must check CEED options before creating function space for coordinates */
4294   {
4295     PetscBool useCeed = PETSC_FALSE, flg;
4296 
4297     PetscCall(PetscOptionsBool("-dm_plex_use_ceed", "Use LibCEED as the FEM backend", "DMPlexSetUseCeed", useCeed, &useCeed, &flg));
4298     if (flg) PetscCall(DMPlexSetUseCeed(dm, useCeed));
4299   }
4300   /* Create coordinate space */
4301   if (created) {
4302     DM_Plex  *mesh   = (DM_Plex *)dm->data;
4303     PetscInt  degree = 1, deg;
4304     PetscInt  height = 0;
4305     DM        cdm;
4306     PetscBool flg;
4307 
4308     PetscCall(PetscOptionsBool("-dm_coord_space", "Use an FEM space for coordinates", "", coordSpace, &coordSpace, &flg));
4309     PetscCall(PetscOptionsInt("-dm_coord_petscspace_degree", "FEM degree for coordinate space", "", degree, &degree, NULL));
4310     PetscCall(DMGetCoordinateDegree_Internal(dm, &deg));
4311     if (coordSpace && deg <= 1) PetscCall(DMPlexCreateCoordinateSpace(dm, degree, PETSC_TRUE, mesh->coordFunc));
4312     PetscCall(DMGetCoordinateDM(dm, &cdm));
4313     if (flg && !coordSpace) {
4314       PetscDS      cds;
4315       PetscObject  obj;
4316       PetscClassId id;
4317 
4318       PetscCall(DMGetDS(cdm, &cds));
4319       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
4320       PetscCall(PetscObjectGetClassId(obj, &id));
4321       if (id == PETSCFE_CLASSID) {
4322         PetscContainer dummy;
4323 
4324         PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &dummy));
4325         PetscCall(PetscObjectSetName((PetscObject)dummy, "coordinates"));
4326         PetscCall(DMSetField(cdm, 0, NULL, (PetscObject)dummy));
4327         PetscCall(PetscContainerDestroy(&dummy));
4328         PetscCall(DMClearDS(cdm));
4329       }
4330       mesh->coordFunc = NULL;
4331     }
4332     PetscCall(PetscOptionsBool("-dm_sparse_localize", "Localize only necessary cells", "", dm->sparseLocalize, &dm->sparseLocalize, &flg));
4333     PetscCall(PetscOptionsInt("-dm_localize_height", "Localize edges and faces in addition to cells", "", height, &height, &flg));
4334     if (flg) PetscCall(DMPlexSetMaxProjectionHeight(cdm, height));
4335     PetscCall(DMLocalizeCoordinates(dm));
4336   }
4337   /* Handle DMPlex refinement */
4338   remap = PETSC_TRUE;
4339   PetscCall(PetscOptionsBoundedInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL, 0));
4340   PetscCall(PetscOptionsBool("-dm_refine_remap", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
4341   PetscCall(PetscOptionsBoundedInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy, 0));
4342   if (refine) PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
4343   if (refine && isHierarchy) {
4344     DM *dms, coarseDM;
4345 
4346     PetscCall(DMGetCoarseDM(dm, &coarseDM));
4347     PetscCall(PetscObjectReference((PetscObject)coarseDM));
4348     PetscCall(PetscMalloc1(refine, &dms));
4349     PetscCall(DMRefineHierarchy(dm, refine, dms));
4350     /* Total hack since we do not pass in a pointer */
4351     PetscCall(DMPlexSwap_Static(dm, dms[refine - 1]));
4352     if (refine == 1) {
4353       PetscCall(DMSetCoarseDM(dm, dms[0]));
4354       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4355     } else {
4356       PetscCall(DMSetCoarseDM(dm, dms[refine - 2]));
4357       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4358       PetscCall(DMSetCoarseDM(dms[0], dms[refine - 1]));
4359       PetscCall(DMPlexSetRegularRefinement(dms[0], PETSC_TRUE));
4360     }
4361     PetscCall(DMSetCoarseDM(dms[refine - 1], coarseDM));
4362     PetscCall(PetscObjectDereference((PetscObject)coarseDM));
4363     /* Free DMs */
4364     for (r = 0; r < refine; ++r) {
4365       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4366       PetscCall(DMDestroy(&dms[r]));
4367     }
4368     PetscCall(PetscFree(dms));
4369   } else {
4370     for (r = 0; r < refine; ++r) {
4371       DM             rdm;
4372       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4373 
4374       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4375       PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
4376       /* Total hack since we do not pass in a pointer */
4377       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4378       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4379       if (coordFunc && remap) {
4380         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4381         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4382       }
4383     }
4384   }
4385   /* Handle DMPlex coarsening */
4386   PetscCall(PetscOptionsBoundedInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL, 0));
4387   PetscCall(PetscOptionsBoundedInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy, 0));
4388   if (coarsen && isHierarchy) {
4389     DM *dms;
4390 
4391     PetscCall(PetscMalloc1(coarsen, &dms));
4392     PetscCall(DMCoarsenHierarchy(dm, coarsen, dms));
4393     /* Free DMs */
4394     for (r = 0; r < coarsen; ++r) {
4395       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4396       PetscCall(DMDestroy(&dms[r]));
4397     }
4398     PetscCall(PetscFree(dms));
4399   } else {
4400     for (r = 0; r < coarsen; ++r) {
4401       DM             cdm;
4402       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4403 
4404       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4405       PetscCall(DMCoarsen(dm, PetscObjectComm((PetscObject)dm), &cdm));
4406       /* Total hack since we do not pass in a pointer */
4407       PetscCall(DMPlexReplace_Internal(dm, &cdm));
4408       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4409       if (coordFunc) {
4410         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4411         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4412       }
4413     }
4414   }
4415   /* Handle ghost cells */
4416   PetscCall(PetscOptionsBool("-dm_plex_create_fv_ghost_cells", "Flag to create finite volume ghost cells on the boundary", "DMCreate", ghostCells, &ghostCells, NULL));
4417   if (ghostCells) {
4418     DM   gdm;
4419     char lname[PETSC_MAX_PATH_LEN];
4420 
4421     lname[0] = '\0';
4422     PetscCall(PetscOptionsString("-dm_plex_fv_ghost_cells_label", "Label name for ghost cells boundary", "DMCreate", lname, lname, sizeof(lname), &flg));
4423     PetscCall(DMPlexConstructGhostCells(dm, flg ? lname : NULL, NULL, &gdm));
4424     PetscCall(DMPlexReplace_Internal(dm, &gdm));
4425   }
4426   /* Handle 1D order */
4427   if (reorder != DMPLEX_REORDER_DEFAULT_FALSE && dim == 1) {
4428     DM           cdm, rdm;
4429     PetscDS      cds;
4430     PetscObject  obj;
4431     PetscClassId id = PETSC_OBJECT_CLASSID;
4432     IS           perm;
4433     PetscInt     Nf;
4434     PetscBool    distributed;
4435 
4436     PetscCall(DMPlexIsDistributed(dm, &distributed));
4437     PetscCall(DMGetCoordinateDM(dm, &cdm));
4438     PetscCall(DMGetDS(cdm, &cds));
4439     PetscCall(PetscDSGetNumFields(cds, &Nf));
4440     if (Nf) {
4441       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
4442       PetscCall(PetscObjectGetClassId(obj, &id));
4443     }
4444     if (!distributed && id != PETSCFE_CLASSID) {
4445       PetscCall(DMPlexGetOrdering1D(dm, &perm));
4446       PetscCall(DMPlexPermute(dm, perm, &rdm));
4447       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4448       PetscCall(ISDestroy(&perm));
4449     }
4450   }
4451 /* Handle */
4452 non_refine:
4453   PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4454   PetscOptionsHeadEnd();
4455   PetscFunctionReturn(PETSC_SUCCESS);
4456 }
4457 
4458 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm, Vec *vec)
4459 {
4460   PetscFunctionBegin;
4461   PetscCall(DMCreateGlobalVector_Section_Private(dm, vec));
4462   /* PetscCall(VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM)); */
4463   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex));
4464   PetscCall(VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void))VecView_Plex_Native));
4465   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex));
4466   PetscCall(VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void))VecLoad_Plex_Native));
4467   PetscFunctionReturn(PETSC_SUCCESS);
4468 }
4469 
4470 static PetscErrorCode DMCreateLocalVector_Plex(DM dm, Vec *vec)
4471 {
4472   PetscFunctionBegin;
4473   PetscCall(DMCreateLocalVector_Section_Private(dm, vec));
4474   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex_Local));
4475   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex_Local));
4476   PetscFunctionReturn(PETSC_SUCCESS);
4477 }
4478 
4479 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
4480 {
4481   PetscInt depth, d;
4482 
4483   PetscFunctionBegin;
4484   PetscCall(DMPlexGetDepth(dm, &depth));
4485   if (depth == 1) {
4486     PetscCall(DMGetDimension(dm, &d));
4487     if (dim == 0) PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4488     else if (dim == d) PetscCall(DMPlexGetDepthStratum(dm, 1, pStart, pEnd));
4489     else {
4490       *pStart = 0;
4491       *pEnd   = 0;
4492     }
4493   } else {
4494     PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4495   }
4496   PetscFunctionReturn(PETSC_SUCCESS);
4497 }
4498 
4499 static PetscErrorCode DMGetNeighbors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
4500 {
4501   PetscSF            sf;
4502   PetscInt           niranks, njranks, n;
4503   const PetscMPIInt *iranks, *jranks;
4504   DM_Plex           *data = (DM_Plex *)dm->data;
4505 
4506   PetscFunctionBegin;
4507   PetscCall(DMGetPointSF(dm, &sf));
4508   if (!data->neighbors) {
4509     PetscCall(PetscSFSetUp(sf));
4510     PetscCall(PetscSFGetRootRanks(sf, &njranks, &jranks, NULL, NULL, NULL));
4511     PetscCall(PetscSFGetLeafRanks(sf, &niranks, &iranks, NULL, NULL));
4512     PetscCall(PetscMalloc1(njranks + niranks + 1, &data->neighbors));
4513     PetscCall(PetscArraycpy(data->neighbors + 1, jranks, njranks));
4514     PetscCall(PetscArraycpy(data->neighbors + njranks + 1, iranks, niranks));
4515     n = njranks + niranks;
4516     PetscCall(PetscSortRemoveDupsMPIInt(&n, data->neighbors + 1));
4517     /* The following cast should never fail: can't have more neighbors than PETSC_MPI_INT_MAX */
4518     PetscCall(PetscMPIIntCast(n, data->neighbors));
4519   }
4520   if (nranks) *nranks = data->neighbors[0];
4521   if (ranks) {
4522     if (data->neighbors[0]) *ranks = data->neighbors + 1;
4523     else *ranks = NULL;
4524   }
4525   PetscFunctionReturn(PETSC_SUCCESS);
4526 }
4527 
4528 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM, DM, Mat, Vec, Vec);
4529 
4530 static PetscErrorCode DMInitialize_Plex(DM dm)
4531 {
4532   PetscFunctionBegin;
4533   dm->ops->view                      = DMView_Plex;
4534   dm->ops->load                      = DMLoad_Plex;
4535   dm->ops->setfromoptions            = DMSetFromOptions_Plex;
4536   dm->ops->clone                     = DMClone_Plex;
4537   dm->ops->setup                     = DMSetUp_Plex;
4538   dm->ops->createlocalsection        = DMCreateLocalSection_Plex;
4539   dm->ops->createdefaultconstraints  = DMCreateDefaultConstraints_Plex;
4540   dm->ops->createglobalvector        = DMCreateGlobalVector_Plex;
4541   dm->ops->createlocalvector         = DMCreateLocalVector_Plex;
4542   dm->ops->getlocaltoglobalmapping   = NULL;
4543   dm->ops->createfieldis             = NULL;
4544   dm->ops->createcoordinatedm        = DMCreateCoordinateDM_Plex;
4545   dm->ops->createcoordinatefield     = DMCreateCoordinateField_Plex;
4546   dm->ops->getcoloring               = NULL;
4547   dm->ops->creatematrix              = DMCreateMatrix_Plex;
4548   dm->ops->createinterpolation       = DMCreateInterpolation_Plex;
4549   dm->ops->createmassmatrix          = DMCreateMassMatrix_Plex;
4550   dm->ops->createmassmatrixlumped    = DMCreateMassMatrixLumped_Plex;
4551   dm->ops->createinjection           = DMCreateInjection_Plex;
4552   dm->ops->refine                    = DMRefine_Plex;
4553   dm->ops->coarsen                   = DMCoarsen_Plex;
4554   dm->ops->refinehierarchy           = DMRefineHierarchy_Plex;
4555   dm->ops->coarsenhierarchy          = DMCoarsenHierarchy_Plex;
4556   dm->ops->extrude                   = DMExtrude_Plex;
4557   dm->ops->globaltolocalbegin        = NULL;
4558   dm->ops->globaltolocalend          = NULL;
4559   dm->ops->localtoglobalbegin        = NULL;
4560   dm->ops->localtoglobalend          = NULL;
4561   dm->ops->destroy                   = DMDestroy_Plex;
4562   dm->ops->createsubdm               = DMCreateSubDM_Plex;
4563   dm->ops->createsuperdm             = DMCreateSuperDM_Plex;
4564   dm->ops->getdimpoints              = DMGetDimPoints_Plex;
4565   dm->ops->locatepoints              = DMLocatePoints_Plex;
4566   dm->ops->projectfunctionlocal      = DMProjectFunctionLocal_Plex;
4567   dm->ops->projectfunctionlabellocal = DMProjectFunctionLabelLocal_Plex;
4568   dm->ops->projectfieldlocal         = DMProjectFieldLocal_Plex;
4569   dm->ops->projectfieldlabellocal    = DMProjectFieldLabelLocal_Plex;
4570   dm->ops->projectbdfieldlabellocal  = DMProjectBdFieldLabelLocal_Plex;
4571   dm->ops->computel2diff             = DMComputeL2Diff_Plex;
4572   dm->ops->computel2gradientdiff     = DMComputeL2GradientDiff_Plex;
4573   dm->ops->computel2fielddiff        = DMComputeL2FieldDiff_Plex;
4574   dm->ops->getneighbors              = DMGetNeighbors_Plex;
4575   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertBoundaryValues_C", DMPlexInsertBoundaryValues_Plex));
4576   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertTimeDerviativeBoundaryValues_C", DMPlexInsertTimeDerivativeBoundaryValues_Plex));
4577   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMSetUpGLVisViewer_C", DMSetUpGLVisViewer_Plex));
4578   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMCreateNeumannOverlap_C", DMCreateNeumannOverlap_Plex));
4579   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4580   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeGetDefault_C", DMPlexDistributeGetDefault_Plex));
4581   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeSetDefault_C", DMPlexDistributeSetDefault_Plex));
4582   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderGetDefault_C", DMPlexReorderGetDefault_Plex));
4583   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderSetDefault_C", DMPlexReorderSetDefault_Plex));
4584   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMInterpolateSolution_C", DMInterpolateSolution_Plex));
4585   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4586   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetOverlap_C", DMPlexSetOverlap_Plex));
4587   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetUseCeed_C", DMPlexGetUseCeed_Plex));
4588   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetUseCeed_C", DMPlexSetUseCeed_Plex));
4589   PetscFunctionReturn(PETSC_SUCCESS);
4590 }
4591 
4592 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
4593 {
4594   DM_Plex *mesh = (DM_Plex *)dm->data;
4595   PetscSF  face_sf;
4596 
4597   PetscFunctionBegin;
4598   mesh->refct++;
4599   (*newdm)->data = mesh;
4600   PetscCall(DMPlexGetIsoperiodicFaceSF(dm, &face_sf));
4601   PetscCall(DMPlexSetIsoperiodicFaceSF(*newdm, face_sf));
4602   PetscCall(PetscObjectChangeTypeName((PetscObject)*newdm, DMPLEX));
4603   PetscCall(DMInitialize_Plex(*newdm));
4604   PetscFunctionReturn(PETSC_SUCCESS);
4605 }
4606 
4607 /*MC
4608   DMPLEX = "plex" - A `DM` object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
4609                     In the local representation, `Vec`s contain all unknowns in the interior and shared boundary. This is
4610                     specified by a PetscSection object. Ownership in the global representation is determined by
4611                     ownership of the underlying `DMPLEX` points. This is specified by another `PetscSection` object.
4612 
4613   Options Database Keys:
4614 + -dm_refine_pre                     - Refine mesh before distribution
4615 + -dm_refine_uniform_pre             - Choose uniform or generator-based refinement
4616 + -dm_refine_volume_limit_pre        - Cell volume limit after pre-refinement using generator
4617 . -dm_distribute                     - Distribute mesh across processes
4618 . -dm_distribute_overlap             - Number of cells to overlap for distribution
4619 . -dm_refine                         - Refine mesh after distribution
4620 . -dm_plex_hash_location             - Use grid hashing for point location
4621 . -dm_plex_hash_box_faces <n,m,p>    - The number of divisions in each direction of the grid hash
4622 . -dm_plex_partition_balance         - Attempt to evenly divide points on partition boundary between processes
4623 . -dm_plex_remesh_bd                 - Allow changes to the boundary on remeshing
4624 . -dm_plex_max_projection_height     - Maximum mesh point height used to project locally
4625 . -dm_plex_regular_refinement        - Use special nested projection algorithm for regular refinement
4626 . -dm_plex_check_all                 - Perform all checks below
4627 . -dm_plex_check_symmetry            - Check that the adjacency information in the mesh is symmetric
4628 . -dm_plex_check_skeleton <celltype> - Check that each cell has the correct number of vertices
4629 . -dm_plex_check_faces <celltype>    - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
4630 . -dm_plex_check_geometry            - Check that cells have positive volume
4631 . -dm_view :mesh.tex:ascii_latex     - View the mesh in LaTeX/TikZ
4632 . -dm_plex_view_scale <num>          - Scale the TikZ
4633 . -dm_plex_print_fem <num>           - View FEM assembly information, such as element vectors and matrices
4634 - -dm_plex_print_fvm <num>           - View FVM assembly information, such as flux updates
4635 
4636   Level: intermediate
4637 
4638 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`, `PetscSection`
4639 M*/
4640 
4641 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
4642 {
4643   DM_Plex *mesh;
4644   PetscInt unit;
4645 
4646   PetscFunctionBegin;
4647   PetscCall(PetscCitationsRegister(PlexCitation, &Plexcite));
4648   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4649   PetscCall(PetscNew(&mesh));
4650   dm->data = mesh;
4651 
4652   mesh->refct = 1;
4653   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection));
4654   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection));
4655   mesh->refinementUniform      = PETSC_TRUE;
4656   mesh->refinementLimit        = -1.0;
4657   mesh->distDefault            = PETSC_TRUE;
4658   mesh->reorderDefault         = DMPLEX_REORDER_DEFAULT_NOTSET;
4659   mesh->distributionName       = NULL;
4660   mesh->interpolated           = DMPLEX_INTERPOLATED_INVALID;
4661   mesh->interpolatedCollective = DMPLEX_INTERPOLATED_INVALID;
4662 
4663   PetscCall(PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner));
4664   mesh->remeshBd = PETSC_FALSE;
4665 
4666   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
4667 
4668   mesh->depthState    = -1;
4669   mesh->celltypeState = -1;
4670   mesh->printTol      = 1.0e-10;
4671 
4672   PetscCall(DMInitialize_Plex(dm));
4673   PetscFunctionReturn(PETSC_SUCCESS);
4674 }
4675 
4676 /*@
4677   DMPlexCreate - Creates a `DMPLEX` object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
4678 
4679   Collective
4680 
4681   Input Parameter:
4682 . comm - The communicator for the `DMPLEX` object
4683 
4684   Output Parameter:
4685 . mesh - The `DMPLEX` object
4686 
4687   Level: beginner
4688 
4689 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMType`, `DMCreate()`, `DMSetType()`
4690 @*/
4691 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
4692 {
4693   PetscFunctionBegin;
4694   PetscAssertPointer(mesh, 2);
4695   PetscCall(DMCreate(comm, mesh));
4696   PetscCall(DMSetType(*mesh, DMPLEX));
4697   PetscFunctionReturn(PETSC_SUCCESS);
4698 }
4699 
4700 /*@C
4701   DMPlexBuildFromCellListParallel - Build distributed `DMPLEX` topology from a list of vertices for each cell (common mesh generator output)
4702 
4703   Collective; No Fortran Support
4704 
4705   Input Parameters:
4706 + dm          - The `DM`
4707 . numCells    - The number of cells owned by this process
4708 . numVertices - The number of vertices to be owned by this process, or `PETSC_DECIDE`
4709 . NVertices   - The global number of vertices, or `PETSC_DETERMINE`
4710 . numCorners  - The number of vertices for each cell
4711 - cells       - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4712 
4713   Output Parameters:
4714 + vertexSF         - (Optional) `PetscSF` describing complete vertex ownership
4715 - verticesAdjSaved - (Optional) vertex adjacency array
4716 
4717   Level: advanced
4718 
4719   Notes:
4720   Two triangles sharing a face
4721 .vb
4722 
4723         2
4724       / | \
4725      /  |  \
4726     /   |   \
4727    0  0 | 1  3
4728     \   |   /
4729      \  |  /
4730       \ | /
4731         1
4732 .ve
4733   would have input
4734 .vb
4735   numCells = 2, numVertices = 4
4736   cells = [0 1 2  1 3 2]
4737 .ve
4738   which would result in the `DMPLEX`
4739 .vb
4740 
4741         4
4742       / | \
4743      /  |  \
4744     /   |   \
4745    2  0 | 1  5
4746     \   |   /
4747      \  |  /
4748       \ | /
4749         3
4750 .ve
4751 
4752   Vertices are implicitly numbered consecutively 0,...,NVertices.
4753   Each rank owns a chunk of numVertices consecutive vertices.
4754   If numVertices is `PETSC_DECIDE`, PETSc will distribute them as evenly as possible using PetscLayout.
4755   If NVertices is `PETSC_DETERMINE` and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
4756   If only NVertices is `PETSC_DETERMINE`, it is computed as the sum of numVertices over all ranks.
4757 
4758   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
4759 
4760 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildCoordinatesFromCellListParallel()`,
4761           `PetscSF`
4762 @*/
4763 PetscErrorCode DMPlexBuildFromCellListParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, const PetscInt cells[], PetscSF *vertexSF, PetscInt **verticesAdjSaved)
4764 {
4765   PetscSF     sfPoint;
4766   PetscLayout layout;
4767   PetscInt    numVerticesAdj, *verticesAdj, *cones, c, p;
4768 
4769   PetscFunctionBegin;
4770   PetscValidLogicalCollectiveInt(dm, NVertices, 4);
4771   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4772   /* Get/check global number of vertices */
4773   {
4774     PetscInt       NVerticesInCells, i;
4775     const PetscInt len = numCells * numCorners;
4776 
4777     /* NVerticesInCells = max(cells) + 1 */
4778     NVerticesInCells = PETSC_MIN_INT;
4779     for (i = 0; i < len; i++)
4780       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4781     ++NVerticesInCells;
4782     PetscCall(MPIU_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
4783 
4784     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
4785     else
4786       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
4787   }
4788   /* Count locally unique vertices */
4789   {
4790     PetscHSetI vhash;
4791     PetscInt   off = 0;
4792 
4793     PetscCall(PetscHSetICreate(&vhash));
4794     for (c = 0; c < numCells; ++c) {
4795       for (p = 0; p < numCorners; ++p) PetscCall(PetscHSetIAdd(vhash, cells[c * numCorners + p]));
4796     }
4797     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
4798     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
4799     else verticesAdj = *verticesAdjSaved;
4800     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
4801     PetscCall(PetscHSetIDestroy(&vhash));
4802     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
4803   }
4804   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
4805   /* Create cones */
4806   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
4807   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
4808   PetscCall(DMSetUp(dm));
4809   PetscCall(DMPlexGetCones(dm, &cones));
4810   for (c = 0; c < numCells; ++c) {
4811     for (p = 0; p < numCorners; ++p) {
4812       const PetscInt gv = cells[c * numCorners + p];
4813       PetscInt       lv;
4814 
4815       /* Positions within verticesAdj form 0-based local vertex numbering;
4816          we need to shift it by numCells to get correct DAG points (cells go first) */
4817       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
4818       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
4819       cones[c * numCorners + p] = lv + numCells;
4820     }
4821   }
4822   /* Build point sf */
4823   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
4824   PetscCall(PetscLayoutSetSize(layout, NVertices));
4825   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
4826   PetscCall(PetscLayoutSetBlockSize(layout, 1));
4827   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
4828   PetscCall(PetscLayoutDestroy(&layout));
4829   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
4830   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
4831   if (dm->sf) {
4832     const char *prefix;
4833 
4834     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
4835     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
4836   }
4837   PetscCall(DMSetPointSF(dm, sfPoint));
4838   PetscCall(PetscSFDestroy(&sfPoint));
4839   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)(*vertexSF), "Vertex Ownership SF"));
4840   /* Fill in the rest of the topology structure */
4841   PetscCall(DMPlexSymmetrize(dm));
4842   PetscCall(DMPlexStratify(dm));
4843   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4844   PetscFunctionReturn(PETSC_SUCCESS);
4845 }
4846 
4847 /*@C
4848   DMPlexBuildCoordinatesFromCellListParallel - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4849 
4850   Collective; No Fortran Support
4851 
4852   Input Parameters:
4853 + dm           - The `DM`
4854 . spaceDim     - The spatial dimension used for coordinates
4855 . sfVert       - `PetscSF` describing complete vertex ownership
4856 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4857 
4858   Level: advanced
4859 
4860 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellListParallel()`
4861 @*/
4862 PetscErrorCode DMPlexBuildCoordinatesFromCellListParallel(DM dm, PetscInt spaceDim, PetscSF sfVert, const PetscReal vertexCoords[])
4863 {
4864   PetscSection coordSection;
4865   Vec          coordinates;
4866   PetscScalar *coords;
4867   PetscInt     numVertices, numVerticesAdj, coordSize, v, vStart, vEnd;
4868 
4869   PetscFunctionBegin;
4870   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4871   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
4872   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4873   PetscCall(DMSetCoordinateDim(dm, spaceDim));
4874   PetscCall(PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL));
4875   PetscCheck(vEnd - vStart == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Supplied sfVert has wrong number of leaves = %" PetscInt_FMT " != %" PetscInt_FMT " = vEnd - vStart", numVerticesAdj, vEnd - vStart);
4876   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4877   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4878   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
4879   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
4880   for (v = vStart; v < vEnd; ++v) {
4881     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
4882     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
4883   }
4884   PetscCall(PetscSectionSetUp(coordSection));
4885   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4886   PetscCall(VecCreate(PetscObjectComm((PetscObject)dm), &coordinates));
4887   PetscCall(VecSetBlockSize(coordinates, spaceDim));
4888   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4889   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4890   PetscCall(VecSetType(coordinates, VECSTANDARD));
4891   PetscCall(VecGetArray(coordinates, &coords));
4892   {
4893     MPI_Datatype coordtype;
4894 
4895     /* Need a temp buffer for coords if we have complex/single */
4896     PetscCallMPI(MPI_Type_contiguous(spaceDim, MPIU_SCALAR, &coordtype));
4897     PetscCallMPI(MPI_Type_commit(&coordtype));
4898 #if defined(PETSC_USE_COMPLEX)
4899     {
4900       PetscScalar *svertexCoords;
4901       PetscInt     i;
4902       PetscCall(PetscMalloc1(numVertices * spaceDim, &svertexCoords));
4903       for (i = 0; i < numVertices * spaceDim; i++) svertexCoords[i] = vertexCoords[i];
4904       PetscCall(PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4905       PetscCall(PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4906       PetscCall(PetscFree(svertexCoords));
4907     }
4908 #else
4909     PetscCall(PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4910     PetscCall(PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4911 #endif
4912     PetscCallMPI(MPI_Type_free(&coordtype));
4913   }
4914   PetscCall(VecRestoreArray(coordinates, &coords));
4915   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4916   PetscCall(VecDestroy(&coordinates));
4917   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4918   PetscFunctionReturn(PETSC_SUCCESS);
4919 }
4920 
4921 /*@
4922   DMPlexCreateFromCellListParallelPetsc - Create distributed `DMPLEX` from a list of vertices for each cell (common mesh generator output)
4923 
4924   Collective
4925 
4926   Input Parameters:
4927 + comm         - The communicator
4928 . dim          - The topological dimension of the mesh
4929 . numCells     - The number of cells owned by this process
4930 . numVertices  - The number of vertices owned by this process, or `PETSC_DECIDE`
4931 . NVertices    - The global number of vertices, or `PETSC_DECIDE`
4932 . numCorners   - The number of vertices for each cell
4933 . interpolate  - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4934 . cells        - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4935 . spaceDim     - The spatial dimension used for coordinates
4936 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4937 
4938   Output Parameters:
4939 + dm          - The `DM`
4940 . vertexSF    - (Optional) `PetscSF` describing complete vertex ownership
4941 - verticesAdj - (Optional) vertex adjacency array
4942 
4943   Level: intermediate
4944 
4945   Notes:
4946   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`,
4947   `DMPlexBuildFromCellListParallel()`, `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellListParallel()`
4948 
4949   See `DMPlexBuildFromCellListParallel()` for an example and details about the topology-related parameters.
4950 
4951   See `DMPlexBuildCoordinatesFromCellListParallel()` for details about the geometry-related parameters.
4952 
4953 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
4954 @*/
4955 PetscErrorCode DMPlexCreateFromCellListParallelPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, PetscInt **verticesAdj, DM *dm)
4956 {
4957   PetscSF sfVert;
4958 
4959   PetscFunctionBegin;
4960   PetscCall(DMCreate(comm, dm));
4961   PetscCall(DMSetType(*dm, DMPLEX));
4962   PetscValidLogicalCollectiveInt(*dm, dim, 2);
4963   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
4964   PetscCall(DMSetDimension(*dm, dim));
4965   PetscCall(DMPlexBuildFromCellListParallel(*dm, numCells, numVertices, NVertices, numCorners, cells, &sfVert, verticesAdj));
4966   if (interpolate) {
4967     DM idm;
4968 
4969     PetscCall(DMPlexInterpolate(*dm, &idm));
4970     PetscCall(DMDestroy(dm));
4971     *dm = idm;
4972   }
4973   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
4974   if (vertexSF) *vertexSF = sfVert;
4975   else PetscCall(PetscSFDestroy(&sfVert));
4976   PetscFunctionReturn(PETSC_SUCCESS);
4977 }
4978 
4979 /*@C
4980   DMPlexBuildFromCellList - Build `DMPLEX` topology from a list of vertices for each cell (common mesh generator output)
4981 
4982   Collective; No Fortran Support
4983 
4984   Input Parameters:
4985 + dm          - The `DM`
4986 . numCells    - The number of cells owned by this process
4987 . numVertices - The number of vertices owned by this process, or `PETSC_DETERMINE`
4988 . numCorners  - The number of vertices for each cell
4989 - cells       - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4990 
4991   Level: advanced
4992 
4993   Notes:
4994   Two triangles sharing a face
4995 .vb
4996 
4997         2
4998       / | \
4999      /  |  \
5000     /   |   \
5001    0  0 | 1  3
5002     \   |   /
5003      \  |  /
5004       \ | /
5005         1
5006 .ve
5007   would have input
5008 .vb
5009   numCells = 2, numVertices = 4
5010   cells = [0 1 2  1 3 2]
5011 .ve
5012   which would result in the `DMPLEX`
5013 .vb
5014 
5015         4
5016       / | \
5017      /  |  \
5018     /   |   \
5019    2  0 | 1  5
5020     \   |   /
5021      \  |  /
5022       \ | /
5023         3
5024 .ve
5025 
5026   If numVertices is `PETSC_DETERMINE`, it is computed by PETSc as the maximum vertex index in cells + 1.
5027 
5028 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListPetsc()`
5029 @*/
5030 PetscErrorCode DMPlexBuildFromCellList(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const PetscInt cells[])
5031 {
5032   PetscInt *cones, c, p, dim;
5033 
5034   PetscFunctionBegin;
5035   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5036   PetscCall(DMGetDimension(dm, &dim));
5037   /* Get/check global number of vertices */
5038   {
5039     PetscInt       NVerticesInCells, i;
5040     const PetscInt len = numCells * numCorners;
5041 
5042     /* NVerticesInCells = max(cells) + 1 */
5043     NVerticesInCells = PETSC_MIN_INT;
5044     for (i = 0; i < len; i++)
5045       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
5046     ++NVerticesInCells;
5047 
5048     if (numVertices == PETSC_DECIDE) numVertices = NVerticesInCells;
5049     else
5050       PetscCheck(numVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, numVertices, NVerticesInCells);
5051   }
5052   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
5053   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
5054   PetscCall(DMSetUp(dm));
5055   PetscCall(DMPlexGetCones(dm, &cones));
5056   for (c = 0; c < numCells; ++c) {
5057     for (p = 0; p < numCorners; ++p) cones[c * numCorners + p] = cells[c * numCorners + p] + numCells;
5058   }
5059   PetscCall(DMPlexSymmetrize(dm));
5060   PetscCall(DMPlexStratify(dm));
5061   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5062   PetscFunctionReturn(PETSC_SUCCESS);
5063 }
5064 
5065 /*@C
5066   DMPlexBuildCoordinatesFromCellList - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
5067 
5068   Collective; No Fortran Support
5069 
5070   Input Parameters:
5071 + dm           - The `DM`
5072 . spaceDim     - The spatial dimension used for coordinates
5073 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
5074 
5075   Level: advanced
5076 
5077 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellList()`
5078 @*/
5079 PetscErrorCode DMPlexBuildCoordinatesFromCellList(DM dm, PetscInt spaceDim, const PetscReal vertexCoords[])
5080 {
5081   PetscSection coordSection;
5082   Vec          coordinates;
5083   DM           cdm;
5084   PetscScalar *coords;
5085   PetscInt     v, vStart, vEnd, d;
5086 
5087   PetscFunctionBegin;
5088   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
5089   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
5090   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
5091   PetscCall(DMSetCoordinateDim(dm, spaceDim));
5092   PetscCall(DMGetCoordinateSection(dm, &coordSection));
5093   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5094   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
5095   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
5096   for (v = vStart; v < vEnd; ++v) {
5097     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
5098     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
5099   }
5100   PetscCall(PetscSectionSetUp(coordSection));
5101 
5102   PetscCall(DMGetCoordinateDM(dm, &cdm));
5103   PetscCall(DMCreateLocalVector(cdm, &coordinates));
5104   PetscCall(VecSetBlockSize(coordinates, spaceDim));
5105   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5106   PetscCall(VecGetArrayWrite(coordinates, &coords));
5107   for (v = 0; v < vEnd - vStart; ++v) {
5108     for (d = 0; d < spaceDim; ++d) coords[v * spaceDim + d] = vertexCoords[v * spaceDim + d];
5109   }
5110   PetscCall(VecRestoreArrayWrite(coordinates, &coords));
5111   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
5112   PetscCall(VecDestroy(&coordinates));
5113   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
5114   PetscFunctionReturn(PETSC_SUCCESS);
5115 }
5116 
5117 /*@
5118   DMPlexCreateFromCellListPetsc - Create `DMPLEX` from a list of vertices for each cell (common mesh generator output), but only process 0 takes in the input
5119 
5120   Collective
5121 
5122   Input Parameters:
5123 + comm         - The communicator
5124 . dim          - The topological dimension of the mesh
5125 . numCells     - The number of cells, only on process 0
5126 . numVertices  - The number of vertices owned by this process, or `PETSC_DECIDE`, only on process 0
5127 . numCorners   - The number of vertices for each cell, only on process 0
5128 . interpolate  - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
5129 . cells        - An array of numCells*numCorners numbers, the vertices for each cell, only on process 0
5130 . spaceDim     - The spatial dimension used for coordinates
5131 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex, only on process 0
5132 
5133   Output Parameter:
5134 . dm - The `DM`, which only has points on process 0
5135 
5136   Level: intermediate
5137 
5138   Notes:
5139   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`, `DMPlexBuildFromCellList()`,
5140   `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellList()`
5141 
5142   See `DMPlexBuildFromCellList()` for an example and details about the topology-related parameters.
5143   See `DMPlexBuildCoordinatesFromCellList()` for details about the geometry-related parameters.
5144   See `DMPlexCreateFromCellListParallelPetsc()` for parallel input
5145 
5146 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellList()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
5147 @*/
5148 PetscErrorCode DMPlexCreateFromCellListPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], DM *dm)
5149 {
5150   PetscMPIInt rank;
5151 
5152   PetscFunctionBegin;
5153   PetscCheck(dim, comm, PETSC_ERR_ARG_OUTOFRANGE, "This is not appropriate for 0-dimensional meshes. Consider either creating the DM using DMPlexCreateFromDAG(), by hand, or using DMSwarm.");
5154   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5155   PetscCall(DMCreate(comm, dm));
5156   PetscCall(DMSetType(*dm, DMPLEX));
5157   PetscCall(DMSetDimension(*dm, dim));
5158   if (rank == 0) PetscCall(DMPlexBuildFromCellList(*dm, numCells, numVertices, numCorners, cells));
5159   else PetscCall(DMPlexBuildFromCellList(*dm, 0, 0, 0, NULL));
5160   if (interpolate) {
5161     DM idm;
5162 
5163     PetscCall(DMPlexInterpolate(*dm, &idm));
5164     PetscCall(DMDestroy(dm));
5165     *dm = idm;
5166   }
5167   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, vertexCoords));
5168   else PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, NULL));
5169   PetscFunctionReturn(PETSC_SUCCESS);
5170 }
5171 
5172 /*@
5173   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a `DM`
5174 
5175   Input Parameters:
5176 + dm               - The empty `DM` object, usually from `DMCreate()` and `DMSetDimension()`
5177 . depth            - The depth of the DAG
5178 . numPoints        - Array of size depth + 1 containing the number of points at each `depth`
5179 . coneSize         - The cone size of each point
5180 . cones            - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
5181 . coneOrientations - The orientation of each cone point
5182 - vertexCoords     - An array of `numPoints`[0]*spacedim numbers representing the coordinates of each vertex, with spacedim the value set via `DMSetCoordinateDim()`
5183 
5184   Output Parameter:
5185 . dm - The `DM`
5186 
5187   Level: advanced
5188 
5189   Note:
5190   Two triangles sharing a face would have input
5191 .vb
5192   depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
5193   cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
5194  vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
5195 .ve
5196   which would result in the DMPlex
5197 .vb
5198         4
5199       / | \
5200      /  |  \
5201     /   |   \
5202    2  0 | 1  5
5203     \   |   /
5204      \  |  /
5205       \ | /
5206         3
5207 .ve
5208   Notice that all points are numbered consecutively, unlike `DMPlexCreateFromCellListPetsc()`
5209 
5210 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
5211 @*/
5212 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
5213 {
5214   Vec          coordinates;
5215   PetscSection coordSection;
5216   PetscScalar *coords;
5217   PetscInt     coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
5218 
5219   PetscFunctionBegin;
5220   PetscCall(DMGetDimension(dm, &dim));
5221   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
5222   PetscCheck(dimEmbed >= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Embedding dimension %" PetscInt_FMT " cannot be less than intrinsic dimension %" PetscInt_FMT, dimEmbed, dim);
5223   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
5224   PetscCall(DMPlexSetChart(dm, pStart, pEnd));
5225   for (p = pStart; p < pEnd; ++p) {
5226     PetscCall(DMPlexSetConeSize(dm, p, coneSize[p - pStart]));
5227     if (firstVertex < 0 && !coneSize[p - pStart]) firstVertex = p - pStart;
5228   }
5229   PetscCheck(firstVertex >= 0 || !numPoints[0], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Expected %" PetscInt_FMT " vertices but could not find any", numPoints[0]);
5230   PetscCall(DMSetUp(dm)); /* Allocate space for cones */
5231   for (p = pStart, off = 0; p < pEnd; off += coneSize[p - pStart], ++p) {
5232     PetscCall(DMPlexSetCone(dm, p, &cones[off]));
5233     PetscCall(DMPlexSetConeOrientation(dm, p, &coneOrientations[off]));
5234   }
5235   PetscCall(DMPlexSymmetrize(dm));
5236   PetscCall(DMPlexStratify(dm));
5237   /* Build coordinates */
5238   PetscCall(DMGetCoordinateSection(dm, &coordSection));
5239   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5240   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dimEmbed));
5241   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numPoints[0]));
5242   for (v = firstVertex; v < firstVertex + numPoints[0]; ++v) {
5243     PetscCall(PetscSectionSetDof(coordSection, v, dimEmbed));
5244     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed));
5245   }
5246   PetscCall(PetscSectionSetUp(coordSection));
5247   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
5248   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
5249   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5250   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
5251   PetscCall(VecSetBlockSize(coordinates, dimEmbed));
5252   PetscCall(VecSetType(coordinates, VECSTANDARD));
5253   if (vertexCoords) {
5254     PetscCall(VecGetArray(coordinates, &coords));
5255     for (v = 0; v < numPoints[0]; ++v) {
5256       PetscInt off;
5257 
5258       PetscCall(PetscSectionGetOffset(coordSection, v + firstVertex, &off));
5259       for (d = 0; d < dimEmbed; ++d) coords[off + d] = vertexCoords[v * dimEmbed + d];
5260     }
5261   }
5262   PetscCall(VecRestoreArray(coordinates, &coords));
5263   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
5264   PetscCall(VecDestroy(&coordinates));
5265   PetscFunctionReturn(PETSC_SUCCESS);
5266 }
5267 
5268 /*
5269   DMPlexCreateCellVertexFromFile - Create a `DMPLEX` mesh from a simple cell-vertex file.
5270 
5271   Collective
5272 
5273 + comm        - The MPI communicator
5274 . filename    - Name of the .dat file
5275 - interpolate - Create faces and edges in the mesh
5276 
5277   Output Parameter:
5278 . dm  - The `DM` object representing the mesh
5279 
5280   Level: beginner
5281 
5282   Note:
5283   The format is the simplest possible:
5284 .vb
5285   Ne
5286   v0 v1 ... vk
5287   Nv
5288   x y z marker
5289 .ve
5290 
5291   Developer Note:
5292   Should use a `PetscViewer` not a filename
5293 
5294 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromFile()`, `DMPlexCreateMedFromFile()`, `DMPlexCreateGmsh()`, `DMPlexCreate()`
5295 */
5296 static PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
5297 {
5298   DMLabel      marker;
5299   PetscViewer  viewer;
5300   Vec          coordinates;
5301   PetscSection coordSection;
5302   PetscScalar *coords;
5303   char         line[PETSC_MAX_PATH_LEN];
5304   PetscInt     dim = 3, cdim = 3, coordSize, v, c, d;
5305   PetscMPIInt  rank;
5306   int          snum, Nv, Nc, Ncn, Nl;
5307 
5308   PetscFunctionBegin;
5309   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5310   PetscCall(PetscViewerCreate(comm, &viewer));
5311   PetscCall(PetscViewerSetType(viewer, PETSCVIEWERASCII));
5312   PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
5313   PetscCall(PetscViewerFileSetName(viewer, filename));
5314   if (rank == 0) {
5315     PetscCall(PetscViewerRead(viewer, line, 4, NULL, PETSC_STRING));
5316     snum = sscanf(line, "%d %d %d %d", &Nc, &Nv, &Ncn, &Nl);
5317     PetscCheck(snum == 4, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5318   } else {
5319     Nc = Nv = Ncn = Nl = 0;
5320   }
5321   PetscCall(DMCreate(comm, dm));
5322   PetscCall(DMSetType(*dm, DMPLEX));
5323   PetscCall(DMPlexSetChart(*dm, 0, Nc + Nv));
5324   PetscCall(DMSetDimension(*dm, dim));
5325   PetscCall(DMSetCoordinateDim(*dm, cdim));
5326   /* Read topology */
5327   if (rank == 0) {
5328     char     format[PETSC_MAX_PATH_LEN];
5329     PetscInt cone[8];
5330     int      vbuf[8], v;
5331 
5332     for (c = 0; c < Ncn; ++c) {
5333       format[c * 3 + 0] = '%';
5334       format[c * 3 + 1] = 'd';
5335       format[c * 3 + 2] = ' ';
5336     }
5337     format[Ncn * 3 - 1] = '\0';
5338     for (c = 0; c < Nc; ++c) PetscCall(DMPlexSetConeSize(*dm, c, Ncn));
5339     PetscCall(DMSetUp(*dm));
5340     for (c = 0; c < Nc; ++c) {
5341       PetscCall(PetscViewerRead(viewer, line, Ncn, NULL, PETSC_STRING));
5342       switch (Ncn) {
5343       case 2:
5344         snum = sscanf(line, format, &vbuf[0], &vbuf[1]);
5345         break;
5346       case 3:
5347         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2]);
5348         break;
5349       case 4:
5350         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3]);
5351         break;
5352       case 6:
5353         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5]);
5354         break;
5355       case 8:
5356         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);
5357         break;
5358       default:
5359         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No cell shape with %d vertices", Ncn);
5360       }
5361       PetscCheck(snum == Ncn, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5362       for (v = 0; v < Ncn; ++v) cone[v] = vbuf[v] + Nc;
5363       /* Hexahedra are inverted */
5364       if (Ncn == 8) {
5365         PetscInt tmp = cone[1];
5366         cone[1]      = cone[3];
5367         cone[3]      = tmp;
5368       }
5369       PetscCall(DMPlexSetCone(*dm, c, cone));
5370     }
5371   }
5372   PetscCall(DMPlexSymmetrize(*dm));
5373   PetscCall(DMPlexStratify(*dm));
5374   /* Read coordinates */
5375   PetscCall(DMGetCoordinateSection(*dm, &coordSection));
5376   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5377   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
5378   PetscCall(PetscSectionSetChart(coordSection, Nc, Nc + Nv));
5379   for (v = Nc; v < Nc + Nv; ++v) {
5380     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
5381     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
5382   }
5383   PetscCall(PetscSectionSetUp(coordSection));
5384   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
5385   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
5386   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5387   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
5388   PetscCall(VecSetBlockSize(coordinates, cdim));
5389   PetscCall(VecSetType(coordinates, VECSTANDARD));
5390   PetscCall(VecGetArray(coordinates, &coords));
5391   if (rank == 0) {
5392     char   format[PETSC_MAX_PATH_LEN];
5393     double x[3];
5394     int    l, val[3];
5395 
5396     if (Nl) {
5397       for (l = 0; l < Nl; ++l) {
5398         format[l * 3 + 0] = '%';
5399         format[l * 3 + 1] = 'd';
5400         format[l * 3 + 2] = ' ';
5401       }
5402       format[Nl * 3 - 1] = '\0';
5403       PetscCall(DMCreateLabel(*dm, "marker"));
5404       PetscCall(DMGetLabel(*dm, "marker", &marker));
5405     }
5406     for (v = 0; v < Nv; ++v) {
5407       PetscCall(PetscViewerRead(viewer, line, 3 + Nl, NULL, PETSC_STRING));
5408       snum = sscanf(line, "%lg %lg %lg", &x[0], &x[1], &x[2]);
5409       PetscCheck(snum == 3, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5410       switch (Nl) {
5411       case 0:
5412         snum = 0;
5413         break;
5414       case 1:
5415         snum = sscanf(line, format, &val[0]);
5416         break;
5417       case 2:
5418         snum = sscanf(line, format, &val[0], &val[1]);
5419         break;
5420       case 3:
5421         snum = sscanf(line, format, &val[0], &val[1], &val[2]);
5422         break;
5423       default:
5424         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Request support for %d labels", Nl);
5425       }
5426       PetscCheck(snum == Nl, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5427       for (d = 0; d < cdim; ++d) coords[v * cdim + d] = x[d];
5428       for (l = 0; l < Nl; ++l) PetscCall(DMLabelSetValue(marker, v + Nc, val[l]));
5429     }
5430   }
5431   PetscCall(VecRestoreArray(coordinates, &coords));
5432   PetscCall(DMSetCoordinatesLocal(*dm, coordinates));
5433   PetscCall(VecDestroy(&coordinates));
5434   PetscCall(PetscViewerDestroy(&viewer));
5435   if (interpolate) {
5436     DM      idm;
5437     DMLabel bdlabel;
5438 
5439     PetscCall(DMPlexInterpolate(*dm, &idm));
5440     PetscCall(DMDestroy(dm));
5441     *dm = idm;
5442 
5443     if (!Nl) {
5444       PetscCall(DMCreateLabel(*dm, "marker"));
5445       PetscCall(DMGetLabel(*dm, "marker", &bdlabel));
5446       PetscCall(DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel));
5447       PetscCall(DMPlexLabelComplete(*dm, bdlabel));
5448     }
5449   }
5450   PetscFunctionReturn(PETSC_SUCCESS);
5451 }
5452 
5453 /*@C
5454   DMPlexCreateFromFile - This takes a filename and produces a `DM`
5455 
5456   Collective
5457 
5458   Input Parameters:
5459 + comm        - The communicator
5460 . filename    - A file name
5461 . plexname    - The object name of the resulting `DM`, also used for intra-datafile lookup by some formats
5462 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
5463 
5464   Output Parameter:
5465 . dm - The `DM`
5466 
5467   Options Database Key:
5468 . -dm_plex_create_from_hdf5_xdmf - use the `PETSC_VIEWER_HDF5_XDMF` format for reading HDF5
5469 
5470   Use `-dm_plex_create_ prefix` to pass options to the internal `PetscViewer`, e.g.
5471 $ -dm_plex_create_viewer_hdf5_collective
5472 
5473   Level: beginner
5474 
5475   Notes:
5476   Using `PETSCVIEWERHDF5` type with `PETSC_VIEWER_HDF5_PETSC` format, one can save multiple `DMPLEX`
5477   meshes in a single HDF5 file. This in turn requires one to name the `DMPLEX` object with `PetscObjectSetName()`
5478   before saving it with `DMView()` and before loading it with `DMLoad()` for identification of the mesh object.
5479   The input parameter name is thus used to name the `DMPLEX` object when `DMPlexCreateFromFile()` internally
5480   calls `DMLoad()`. Currently, name is ignored for other viewer types and/or formats.
5481 
5482 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`, `PetscObjectSetName()`, `DMView()`, `DMLoad()`
5483 @*/
5484 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], const char plexname[], PetscBool interpolate, DM *dm)
5485 {
5486   const char  extGmsh[]      = ".msh";
5487   const char  extGmsh2[]     = ".msh2";
5488   const char  extGmsh4[]     = ".msh4";
5489   const char  extCGNS[]      = ".cgns";
5490   const char  extExodus[]    = ".exo";
5491   const char  extExodus_e[]  = ".e";
5492   const char  extGenesis[]   = ".gen";
5493   const char  extFluent[]    = ".cas";
5494   const char  extHDF5[]      = ".h5";
5495   const char  extXDMFHDF5[]  = ".xdmf.h5";
5496   const char  extMed[]       = ".med";
5497   const char  extPLY[]       = ".ply";
5498   const char  extEGADSLite[] = ".egadslite";
5499   const char  extEGADS[]     = ".egads";
5500   const char  extIGES[]      = ".igs";
5501   const char  extSTEP[]      = ".stp";
5502   const char  extCV[]        = ".dat";
5503   size_t      len;
5504   PetscBool   isGmsh, isGmsh2, isGmsh4, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isMed, isPLY, isEGADSLite, isEGADS, isIGES, isSTEP, isCV, isXDMFHDF5;
5505   PetscMPIInt rank;
5506 
5507   PetscFunctionBegin;
5508   PetscAssertPointer(filename, 2);
5509   if (plexname) PetscAssertPointer(plexname, 3);
5510   PetscAssertPointer(dm, 5);
5511   PetscCall(DMInitializePackage());
5512   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5513   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5514   PetscCall(PetscStrlen(filename, &len));
5515   PetscCheck(len, comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
5516 
5517 #define CheckExtension(extension__, is_extension__) \
5518   do { \
5519     PetscAssert(sizeof(extension__), comm, PETSC_ERR_PLIB, "Zero-size extension: %s", extension__); \
5520     /* don't count the null-terminator at the end */ \
5521     const size_t ext_len = sizeof(extension__) - 1; \
5522     if (len < ext_len) { \
5523       is_extension__ = PETSC_FALSE; \
5524     } else { \
5525       PetscCall(PetscStrncmp(filename + len - ext_len, extension__, ext_len, &is_extension__)); \
5526     } \
5527   } while (0)
5528 
5529   CheckExtension(extGmsh, isGmsh);
5530   CheckExtension(extGmsh2, isGmsh2);
5531   CheckExtension(extGmsh4, isGmsh4);
5532   CheckExtension(extCGNS, isCGNS);
5533   CheckExtension(extExodus, isExodus);
5534   if (!isExodus) CheckExtension(extExodus_e, isExodus);
5535   CheckExtension(extGenesis, isGenesis);
5536   CheckExtension(extFluent, isFluent);
5537   CheckExtension(extHDF5, isHDF5);
5538   CheckExtension(extMed, isMed);
5539   CheckExtension(extPLY, isPLY);
5540   CheckExtension(extEGADSLite, isEGADSLite);
5541   CheckExtension(extEGADS, isEGADS);
5542   CheckExtension(extIGES, isIGES);
5543   CheckExtension(extSTEP, isSTEP);
5544   CheckExtension(extCV, isCV);
5545   CheckExtension(extXDMFHDF5, isXDMFHDF5);
5546 
5547 #undef CheckExtension
5548 
5549   if (isGmsh || isGmsh2 || isGmsh4) {
5550     PetscCall(DMPlexCreateGmshFromFile(comm, filename, interpolate, dm));
5551   } else if (isCGNS) {
5552     PetscCall(DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm));
5553   } else if (isExodus || isGenesis) {
5554     PetscCall(DMPlexCreateExodusFromFile(comm, filename, interpolate, dm));
5555   } else if (isFluent) {
5556     PetscCall(DMPlexCreateFluentFromFile(comm, filename, interpolate, dm));
5557   } else if (isHDF5) {
5558     PetscViewer viewer;
5559 
5560     /* PETSC_VIEWER_HDF5_XDMF is used if the filename ends with .xdmf.h5, or if -dm_plex_create_from_hdf5_xdmf option is present */
5561     PetscCall(PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &isXDMFHDF5, NULL));
5562     PetscCall(PetscViewerCreate(comm, &viewer));
5563     PetscCall(PetscViewerSetType(viewer, PETSCVIEWERHDF5));
5564     PetscCall(PetscViewerSetOptionsPrefix(viewer, "dm_plex_create_"));
5565     PetscCall(PetscViewerSetFromOptions(viewer));
5566     PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
5567     PetscCall(PetscViewerFileSetName(viewer, filename));
5568 
5569     PetscCall(DMCreate(comm, dm));
5570     PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5571     PetscCall(DMSetType(*dm, DMPLEX));
5572     if (isXDMFHDF5) PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF));
5573     PetscCall(DMLoad(*dm, viewer));
5574     if (isXDMFHDF5) PetscCall(PetscViewerPopFormat(viewer));
5575     PetscCall(PetscViewerDestroy(&viewer));
5576 
5577     if (interpolate) {
5578       DM idm;
5579 
5580       PetscCall(DMPlexInterpolate(*dm, &idm));
5581       PetscCall(DMDestroy(dm));
5582       *dm = idm;
5583     }
5584   } else if (isMed) {
5585     PetscCall(DMPlexCreateMedFromFile(comm, filename, interpolate, dm));
5586   } else if (isPLY) {
5587     PetscCall(DMPlexCreatePLYFromFile(comm, filename, interpolate, dm));
5588   } else if (isEGADSLite || isEGADS || isIGES || isSTEP) {
5589     if (isEGADSLite) PetscCall(DMPlexCreateEGADSLiteFromFile(comm, filename, dm));
5590     else PetscCall(DMPlexCreateEGADSFromFile(comm, filename, dm));
5591     if (!interpolate) {
5592       DM udm;
5593 
5594       PetscCall(DMPlexUninterpolate(*dm, &udm));
5595       PetscCall(DMDestroy(dm));
5596       *dm = udm;
5597     }
5598   } else if (isCV) {
5599     PetscCall(DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm));
5600   } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
5601   PetscCall(PetscStrlen(plexname, &len));
5602   if (len) PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5603   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5604   PetscFunctionReturn(PETSC_SUCCESS);
5605 }
5606 
5607 /*@C
5608   DMPlexCreateEphemeral - This takes a `DMPlexTransform` and a base `DMPlex` and produces an ephemeral `DM`, meaning one that is created on the fly in response to queries.
5609 
5610   Input Parameters:
5611 + tr     - The `DMPlexTransform`
5612 - prefix - An options prefix, or NULL
5613 
5614   Output Parameter:
5615 . dm - The `DM`
5616 
5617   Level: beginner
5618 
5619   Notes:
5620   An emphemeral mesh is one that is not stored concretely, as in the default `DMPLEX` implementation, but rather is produced on the fly in response to queries, using information from the transform and the base mesh.
5621 
5622 .seealso: `DMPlexCreateFromFile`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
5623 @*/
5624 PetscErrorCode DMPlexCreateEphemeral(DMPlexTransform tr, const char prefix[], DM *dm)
5625 {
5626   DM           bdm, bcdm, cdm;
5627   Vec          coordinates, coordinatesNew;
5628   PetscSection cs;
5629   PetscInt     dim, cdim, Nl;
5630 
5631   PetscFunctionBegin;
5632   PetscCall(DMCreate(PetscObjectComm((PetscObject)tr), dm));
5633   PetscCall(DMSetType(*dm, DMPLEX));
5634   ((DM_Plex *)(*dm)->data)->interpolated = DMPLEX_INTERPOLATED_FULL;
5635   // Handle coordinates
5636   PetscCall(DMPlexTransformGetDM(tr, &bdm));
5637   PetscCall(DMGetCoordinateDim(bdm, &cdim));
5638   PetscCall(DMSetCoordinateDim(*dm, cdim));
5639   PetscCall(DMGetDimension(bdm, &dim));
5640   PetscCall(DMSetDimension(*dm, dim));
5641   PetscCall(DMGetCoordinateDM(bdm, &bcdm));
5642   PetscCall(DMGetCoordinateDM(*dm, &cdm));
5643   PetscCall(DMCopyDisc(bcdm, cdm));
5644   PetscCall(DMGetLocalSection(cdm, &cs));
5645   PetscCall(PetscSectionSetNumFields(cs, 1));
5646   PetscCall(PetscSectionSetFieldComponents(cs, 0, cdim));
5647   PetscCall(DMGetCoordinatesLocal(bdm, &coordinates));
5648   PetscCall(VecDuplicate(coordinates, &coordinatesNew));
5649   PetscCall(VecCopy(coordinates, coordinatesNew));
5650   PetscCall(DMSetCoordinatesLocal(*dm, coordinatesNew));
5651   PetscCall(VecDestroy(&coordinatesNew));
5652 
5653   PetscCall(PetscObjectReference((PetscObject)tr));
5654   PetscCall(DMPlexTransformDestroy(&((DM_Plex *)(*dm)->data)->tr));
5655   ((DM_Plex *)(*dm)->data)->tr = tr;
5656   PetscCall(DMPlexDistributeSetDefault(*dm, PETSC_FALSE));
5657   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)*dm, prefix));
5658   PetscCall(DMSetFromOptions(*dm));
5659 
5660   PetscCall(DMGetNumLabels(bdm, &Nl));
5661   for (PetscInt l = 0; l < Nl; ++l) {
5662     DMLabel     label, labelNew;
5663     const char *lname;
5664     PetscBool   isDepth, isCellType;
5665 
5666     PetscCall(DMGetLabelName(bdm, l, &lname));
5667     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
5668     if (isDepth) continue;
5669     PetscCall(PetscStrcmp(lname, "celltype", &isCellType));
5670     if (isCellType) continue;
5671     PetscCall(DMCreateLabel(*dm, lname));
5672     PetscCall(DMGetLabel(bdm, lname, &label));
5673     PetscCall(DMGetLabel(*dm, lname, &labelNew));
5674     PetscCall(DMLabelSetType(labelNew, DMLABELEPHEMERAL));
5675     PetscCall(DMLabelEphemeralSetLabel(labelNew, label));
5676     PetscCall(DMLabelEphemeralSetTransform(labelNew, tr));
5677     PetscCall(DMLabelSetUp(labelNew));
5678   }
5679   PetscFunctionReturn(PETSC_SUCCESS);
5680 }
5681