xref: /petsc/src/dm/impls/plex/plexcreate.c (revision a8d3b578d34eb8d5be1b3eaad4a98fb37a558ece)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h> /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>   /*I   "petscdmplex.h"   I*/
4 #include <petscsf.h>
5 #include <petscdmplextransform.h>
6 #include <petsc/private/kernels/blockmatmult.h>
7 #include <petsc/private/kernels/blockinvert.h>
8 
9 PetscLogEvent DMPLEX_CreateFromFile, DMPLEX_BuildFromCellList, DMPLEX_BuildCoordinatesFromCellList;
10 
11 /* External function declarations here */
12 static PetscErrorCode DMInitialize_Plex(DM dm);
13 
14 /* This copies internal things in the Plex structure that we generally want when making a new, related Plex */
15 PetscErrorCode DMPlexCopy_Internal(DM dmin, PetscBool copyPeriodicity, PetscBool copyOverlap, DM dmout)
16 {
17   const PetscReal         *maxCell, *Lstart, *L;
18   PetscBool                dist;
19   DMPlexReorderDefaultFlag reorder;
20 
21   PetscFunctionBegin;
22   if (copyPeriodicity) {
23     PetscCall(DMGetPeriodicity(dmin, &maxCell, &Lstart, &L));
24     PetscCall(DMSetPeriodicity(dmout, maxCell, Lstart, L));
25   }
26   PetscCall(DMPlexDistributeGetDefault(dmin, &dist));
27   PetscCall(DMPlexDistributeSetDefault(dmout, dist));
28   PetscCall(DMPlexReorderGetDefault(dmin, &reorder));
29   PetscCall(DMPlexReorderSetDefault(dmout, reorder));
30   ((DM_Plex *)dmout->data)->useHashLocation = ((DM_Plex *)dmin->data)->useHashLocation;
31   if (copyOverlap) PetscCall(DMPlexSetOverlap_Plex(dmout, dmin, 0));
32   PetscFunctionReturn(0);
33 }
34 
35 /* Replace dm with the contents of ndm, and then destroy ndm
36    - Share the DM_Plex structure
37    - Share the coordinates
38    - Share the SF
39 */
40 PetscErrorCode DMPlexReplace_Internal(DM dm, DM *ndm)
41 {
42   PetscSF          sf;
43   DM               dmNew = *ndm, coordDM, coarseDM;
44   Vec              coords;
45   const PetscReal *maxCell, *Lstart, *L;
46   PetscInt         dim, cdim;
47 
48   PetscFunctionBegin;
49   if (dm == dmNew) {
50     PetscCall(DMDestroy(ndm));
51     PetscFunctionReturn(0);
52   }
53   dm->setupcalled = dmNew->setupcalled;
54   PetscCall(DMGetDimension(dmNew, &dim));
55   PetscCall(DMSetDimension(dm, dim));
56   PetscCall(DMGetCoordinateDim(dmNew, &cdim));
57   PetscCall(DMSetCoordinateDim(dm, cdim));
58   PetscCall(DMGetPointSF(dmNew, &sf));
59   PetscCall(DMSetPointSF(dm, sf));
60   PetscCall(DMGetCoordinateDM(dmNew, &coordDM));
61   PetscCall(DMGetCoordinatesLocal(dmNew, &coords));
62   PetscCall(DMSetCoordinateDM(dm, coordDM));
63   PetscCall(DMSetCoordinatesLocal(dm, coords));
64   PetscCall(DMGetCellCoordinateDM(dmNew, &coordDM));
65   PetscCall(DMGetCellCoordinatesLocal(dmNew, &coords));
66   PetscCall(DMSetCellCoordinateDM(dm, coordDM));
67   PetscCall(DMSetCellCoordinatesLocal(dm, coords));
68   /* Do not want to create the coordinate field if it does not already exist, so do not call DMGetCoordinateField() */
69   PetscCall(DMFieldDestroy(&dm->coordinates[0].field));
70   dm->coordinates[0].field            = dmNew->coordinates[0].field;
71   ((DM_Plex *)dmNew->data)->coordFunc = ((DM_Plex *)dm->data)->coordFunc;
72   PetscCall(DMGetPeriodicity(dmNew, &maxCell, &Lstart, &L));
73   PetscCall(DMSetPeriodicity(dm, maxCell, Lstart, L));
74   PetscCall(DMDestroy_Plex(dm));
75   PetscCall(DMInitialize_Plex(dm));
76   dm->data = dmNew->data;
77   ((DM_Plex *)dmNew->data)->refct++;
78   PetscCall(DMDestroyLabelLinkList_Internal(dm));
79   PetscCall(DMCopyLabels(dmNew, dm, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL));
80   PetscCall(DMGetCoarseDM(dmNew, &coarseDM));
81   PetscCall(DMSetCoarseDM(dm, coarseDM));
82   PetscCall(DMDestroy(ndm));
83   PetscFunctionReturn(0);
84 }
85 
86 /* Swap dm with the contents of dmNew
87    - Swap the DM_Plex structure
88    - Swap the coordinates
89    - Swap the point PetscSF
90 */
91 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
92 {
93   DM          coordDMA, coordDMB;
94   Vec         coordsA, coordsB;
95   PetscSF     sfA, sfB;
96   DMField     fieldTmp;
97   void       *tmp;
98   DMLabelLink listTmp;
99   DMLabel     depthTmp;
100   PetscInt    tmpI;
101 
102   PetscFunctionBegin;
103   if (dmA == dmB) PetscFunctionReturn(0);
104   PetscCall(DMGetPointSF(dmA, &sfA));
105   PetscCall(DMGetPointSF(dmB, &sfB));
106   PetscCall(PetscObjectReference((PetscObject)sfA));
107   PetscCall(DMSetPointSF(dmA, sfB));
108   PetscCall(DMSetPointSF(dmB, sfA));
109   PetscCall(PetscObjectDereference((PetscObject)sfA));
110 
111   PetscCall(DMGetCoordinateDM(dmA, &coordDMA));
112   PetscCall(DMGetCoordinateDM(dmB, &coordDMB));
113   PetscCall(PetscObjectReference((PetscObject)coordDMA));
114   PetscCall(DMSetCoordinateDM(dmA, coordDMB));
115   PetscCall(DMSetCoordinateDM(dmB, coordDMA));
116   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
117 
118   PetscCall(DMGetCoordinatesLocal(dmA, &coordsA));
119   PetscCall(DMGetCoordinatesLocal(dmB, &coordsB));
120   PetscCall(PetscObjectReference((PetscObject)coordsA));
121   PetscCall(DMSetCoordinatesLocal(dmA, coordsB));
122   PetscCall(DMSetCoordinatesLocal(dmB, coordsA));
123   PetscCall(PetscObjectDereference((PetscObject)coordsA));
124 
125   PetscCall(DMGetCellCoordinateDM(dmA, &coordDMA));
126   PetscCall(DMGetCellCoordinateDM(dmB, &coordDMB));
127   PetscCall(PetscObjectReference((PetscObject)coordDMA));
128   PetscCall(DMSetCellCoordinateDM(dmA, coordDMB));
129   PetscCall(DMSetCellCoordinateDM(dmB, coordDMA));
130   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
131 
132   PetscCall(DMGetCellCoordinatesLocal(dmA, &coordsA));
133   PetscCall(DMGetCellCoordinatesLocal(dmB, &coordsB));
134   PetscCall(PetscObjectReference((PetscObject)coordsA));
135   PetscCall(DMSetCellCoordinatesLocal(dmA, coordsB));
136   PetscCall(DMSetCellCoordinatesLocal(dmB, coordsA));
137   PetscCall(PetscObjectDereference((PetscObject)coordsA));
138 
139   fieldTmp                  = dmA->coordinates[0].field;
140   dmA->coordinates[0].field = dmB->coordinates[0].field;
141   dmB->coordinates[0].field = fieldTmp;
142   fieldTmp                  = dmA->coordinates[1].field;
143   dmA->coordinates[1].field = dmB->coordinates[1].field;
144   dmB->coordinates[1].field = fieldTmp;
145   tmp                       = dmA->data;
146   dmA->data                 = dmB->data;
147   dmB->data                 = tmp;
148   listTmp                   = dmA->labels;
149   dmA->labels               = dmB->labels;
150   dmB->labels               = listTmp;
151   depthTmp                  = dmA->depthLabel;
152   dmA->depthLabel           = dmB->depthLabel;
153   dmB->depthLabel           = depthTmp;
154   depthTmp                  = dmA->celltypeLabel;
155   dmA->celltypeLabel        = dmB->celltypeLabel;
156   dmB->celltypeLabel        = depthTmp;
157   tmpI                      = dmA->levelup;
158   dmA->levelup              = dmB->levelup;
159   dmB->levelup              = tmpI;
160   PetscFunctionReturn(0);
161 }
162 
163 static PetscErrorCode DMPlexInterpolateInPlace_Internal(DM dm)
164 {
165   DM idm;
166 
167   PetscFunctionBegin;
168   PetscCall(DMPlexInterpolate(dm, &idm));
169   PetscCall(DMPlexCopyCoordinates(dm, idm));
170   PetscCall(DMPlexReplace_Internal(dm, &idm));
171   PetscFunctionReturn(0);
172 }
173 
174 /*@C
175   DMPlexCreateCoordinateSpace - Creates a finite element space for the coordinates
176 
177   Collective on dm
178 
179   Input Parameters:
180 + DM        - The DM
181 . degree    - The degree of the finite element or PETSC_DECIDE
182 - coordFunc - An optional function to map new points from refinement to the surface
183 
184   Level: advanced
185 
186 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `PetscPointFunc`, `PetscFECreateLagrange()`, `DMGetCoordinateDM()`
187 @*/
188 PetscErrorCode DMPlexCreateCoordinateSpace(DM dm, PetscInt degree, PetscPointFunc coordFunc)
189 {
190   DM_Plex     *mesh = (DM_Plex *)dm->data;
191   DM           cdm;
192   PetscDS      cds;
193   PetscFE      fe;
194   PetscClassId id;
195 
196   PetscFunctionBegin;
197   PetscCall(DMGetCoordinateDM(dm, &cdm));
198   PetscCall(DMGetDS(cdm, &cds));
199   PetscCall(PetscDSGetDiscretization(cds, 0, (PetscObject *)&fe));
200   PetscCall(PetscObjectGetClassId((PetscObject)fe, &id));
201   if (id != PETSCFE_CLASSID) {
202     PetscBool simplex;
203     PetscInt  dim, dE, qorder;
204 
205     PetscCall(DMGetDimension(dm, &dim));
206     PetscCall(DMGetCoordinateDim(dm, &dE));
207     qorder = degree;
208     PetscObjectOptionsBegin((PetscObject)cdm);
209     PetscCall(PetscOptionsBoundedInt("-coord_dm_default_quadrature_order", "Quadrature order is one less than quadrature points per edge", "DMPlexCreateCoordinateSpace", qorder, &qorder, NULL, 0));
210     PetscOptionsEnd();
211     if (degree == PETSC_DECIDE) fe = NULL;
212     else {
213       PetscCall(DMPlexIsSimplex(dm, &simplex));
214       PetscCall(PetscFECreateLagrange(PETSC_COMM_SELF, dim, dE, simplex, degree, qorder, &fe));
215     }
216     PetscCall(DMProjectCoordinates(dm, fe));
217     PetscCall(PetscFEDestroy(&fe));
218   }
219   mesh->coordFunc = coordFunc;
220   PetscFunctionReturn(0);
221 }
222 
223 /*@
224   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
225 
226   Collective
227 
228   Input Parameters:
229 + comm - The communicator for the `DM` object
230 . dim - The spatial dimension
231 . simplex - Flag for simplicial cells, otherwise they are tensor product cells
232 . interpolate - Flag to create intermediate mesh pieces (edges, faces)
233 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
234 
235   Output Parameter:
236 . dm - The `DM` object
237 
238   Level: beginner
239 
240 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMSetType()`, `DMCreate()`
241 @*/
242 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscReal refinementLimit, DM *newdm)
243 {
244   DM          dm;
245   PetscMPIInt rank;
246 
247   PetscFunctionBegin;
248   PetscCall(DMCreate(comm, &dm));
249   PetscCall(DMSetType(dm, DMPLEX));
250   PetscCall(DMSetDimension(dm, dim));
251   PetscCallMPI(MPI_Comm_rank(comm, &rank));
252   switch (dim) {
253   case 2:
254     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "triangular"));
255     else PetscCall(PetscObjectSetName((PetscObject)dm, "quadrilateral"));
256     break;
257   case 3:
258     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "tetrahedral"));
259     else PetscCall(PetscObjectSetName((PetscObject)dm, "hexahedral"));
260     break;
261   default:
262     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
263   }
264   if (rank) {
265     PetscInt numPoints[2] = {0, 0};
266     PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL));
267   } else {
268     switch (dim) {
269     case 2:
270       if (simplex) {
271         PetscInt    numPoints[2]        = {4, 2};
272         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
273         PetscInt    cones[6]            = {2, 3, 4, 5, 4, 3};
274         PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
275         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
276 
277         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
278       } else {
279         PetscInt    numPoints[2]        = {6, 2};
280         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
281         PetscInt    cones[8]            = {2, 3, 4, 5, 3, 6, 7, 4};
282         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
283         PetscScalar vertexCoords[12]    = {-1.0, -0.5, 0.0, -0.5, 0.0, 0.5, -1.0, 0.5, 1.0, -0.5, 1.0, 0.5};
284 
285         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
286       }
287       break;
288     case 3:
289       if (simplex) {
290         PetscInt    numPoints[2]        = {5, 2};
291         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
292         PetscInt    cones[8]            = {4, 3, 5, 2, 5, 3, 4, 6};
293         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
294         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0};
295 
296         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
297       } else {
298         PetscInt    numPoints[2]         = {12, 2};
299         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
300         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9, 5, 4, 10, 11, 7, 12, 13, 8};
301         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
302         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5, -1.0, 0.5, -0.5, 0.0, 0.5, -0.5, 0.0, -0.5, -0.5, -1.0, -0.5, 0.5, 0.0, -0.5, 0.5, 0.0, 0.5, 0.5, -1.0, 0.5, 0.5, 1.0, 0.5, -0.5, 1.0, -0.5, -0.5, 1.0, -0.5, 0.5, 1.0, 0.5, 0.5};
303 
304         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
305       }
306       break;
307     default:
308       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
309     }
310   }
311   *newdm = dm;
312   if (refinementLimit > 0.0) {
313     DM          rdm;
314     const char *name;
315 
316     PetscCall(DMPlexSetRefinementUniform(*newdm, PETSC_FALSE));
317     PetscCall(DMPlexSetRefinementLimit(*newdm, refinementLimit));
318     PetscCall(DMRefine(*newdm, comm, &rdm));
319     PetscCall(PetscObjectGetName((PetscObject)*newdm, &name));
320     PetscCall(PetscObjectSetName((PetscObject)rdm, name));
321     PetscCall(DMDestroy(newdm));
322     *newdm = rdm;
323   }
324   if (interpolate) {
325     DM idm;
326 
327     PetscCall(DMPlexInterpolate(*newdm, &idm));
328     PetscCall(DMDestroy(newdm));
329     *newdm = idm;
330   }
331   PetscFunctionReturn(0);
332 }
333 
334 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
335 {
336   const PetscInt numVertices    = 2;
337   PetscInt       markerRight    = 1;
338   PetscInt       markerLeft     = 1;
339   PetscBool      markerSeparate = PETSC_FALSE;
340   Vec            coordinates;
341   PetscSection   coordSection;
342   PetscScalar   *coords;
343   PetscInt       coordSize;
344   PetscMPIInt    rank;
345   PetscInt       cdim = 1, v;
346 
347   PetscFunctionBegin;
348   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
349   if (markerSeparate) {
350     markerRight = 2;
351     markerLeft  = 1;
352   }
353   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
354   if (rank == 0) {
355     PetscCall(DMPlexSetChart(dm, 0, numVertices));
356     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
357     PetscCall(DMSetLabelValue(dm, "marker", 0, markerLeft));
358     PetscCall(DMSetLabelValue(dm, "marker", 1, markerRight));
359   }
360   PetscCall(DMPlexSymmetrize(dm));
361   PetscCall(DMPlexStratify(dm));
362   /* Build coordinates */
363   PetscCall(DMSetCoordinateDim(dm, cdim));
364   PetscCall(DMGetCoordinateSection(dm, &coordSection));
365   PetscCall(PetscSectionSetNumFields(coordSection, 1));
366   PetscCall(PetscSectionSetChart(coordSection, 0, numVertices));
367   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
368   for (v = 0; v < numVertices; ++v) {
369     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
370     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
371   }
372   PetscCall(PetscSectionSetUp(coordSection));
373   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
374   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
375   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
376   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
377   PetscCall(VecSetBlockSize(coordinates, cdim));
378   PetscCall(VecSetType(coordinates, VECSTANDARD));
379   PetscCall(VecGetArray(coordinates, &coords));
380   coords[0] = lower[0];
381   coords[1] = upper[0];
382   PetscCall(VecRestoreArray(coordinates, &coords));
383   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
384   PetscCall(VecDestroy(&coordinates));
385   PetscFunctionReturn(0);
386 }
387 
388 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
389 {
390   const PetscInt numVertices    = (edges[0] + 1) * (edges[1] + 1);
391   const PetscInt numEdges       = edges[0] * (edges[1] + 1) + (edges[0] + 1) * edges[1];
392   PetscInt       markerTop      = 1;
393   PetscInt       markerBottom   = 1;
394   PetscInt       markerRight    = 1;
395   PetscInt       markerLeft     = 1;
396   PetscBool      markerSeparate = PETSC_FALSE;
397   Vec            coordinates;
398   PetscSection   coordSection;
399   PetscScalar   *coords;
400   PetscInt       coordSize;
401   PetscMPIInt    rank;
402   PetscInt       v, vx, vy;
403 
404   PetscFunctionBegin;
405   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
406   if (markerSeparate) {
407     markerTop    = 3;
408     markerBottom = 1;
409     markerRight  = 2;
410     markerLeft   = 4;
411   }
412   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
413   if (rank == 0) {
414     PetscInt e, ex, ey;
415 
416     PetscCall(DMPlexSetChart(dm, 0, numEdges + numVertices));
417     for (e = 0; e < numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
418     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
419     for (vx = 0; vx <= edges[0]; vx++) {
420       for (ey = 0; ey < edges[1]; ey++) {
421         PetscInt edge   = vx * edges[1] + ey + edges[0] * (edges[1] + 1);
422         PetscInt vertex = ey * (edges[0] + 1) + vx + numEdges;
423         PetscInt cone[2];
424 
425         cone[0] = vertex;
426         cone[1] = vertex + edges[0] + 1;
427         PetscCall(DMPlexSetCone(dm, edge, cone));
428         if (vx == edges[0]) {
429           PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
430           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
431           if (ey == edges[1] - 1) {
432             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
433             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerRight));
434           }
435         } else if (vx == 0) {
436           PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
437           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
438           if (ey == edges[1] - 1) {
439             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
440             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft));
441           }
442         }
443       }
444     }
445     for (vy = 0; vy <= edges[1]; vy++) {
446       for (ex = 0; ex < edges[0]; ex++) {
447         PetscInt edge   = vy * edges[0] + ex;
448         PetscInt vertex = vy * (edges[0] + 1) + ex + numEdges;
449         PetscInt cone[2];
450 
451         cone[0] = vertex;
452         cone[1] = vertex + 1;
453         PetscCall(DMPlexSetCone(dm, edge, cone));
454         if (vy == edges[1]) {
455           PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
456           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
457           if (ex == edges[0] - 1) {
458             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
459             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerTop));
460           }
461         } else if (vy == 0) {
462           PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
463           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
464           if (ex == edges[0] - 1) {
465             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
466             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom));
467           }
468         }
469       }
470     }
471   }
472   PetscCall(DMPlexSymmetrize(dm));
473   PetscCall(DMPlexStratify(dm));
474   /* Build coordinates */
475   PetscCall(DMSetCoordinateDim(dm, 2));
476   PetscCall(DMGetCoordinateSection(dm, &coordSection));
477   PetscCall(PetscSectionSetNumFields(coordSection, 1));
478   PetscCall(PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices));
479   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 2));
480   for (v = numEdges; v < numEdges + numVertices; ++v) {
481     PetscCall(PetscSectionSetDof(coordSection, v, 2));
482     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 2));
483   }
484   PetscCall(PetscSectionSetUp(coordSection));
485   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
486   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
487   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
488   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
489   PetscCall(VecSetBlockSize(coordinates, 2));
490   PetscCall(VecSetType(coordinates, VECSTANDARD));
491   PetscCall(VecGetArray(coordinates, &coords));
492   for (vy = 0; vy <= edges[1]; ++vy) {
493     for (vx = 0; vx <= edges[0]; ++vx) {
494       coords[(vy * (edges[0] + 1) + vx) * 2 + 0] = lower[0] + ((upper[0] - lower[0]) / edges[0]) * vx;
495       coords[(vy * (edges[0] + 1) + vx) * 2 + 1] = lower[1] + ((upper[1] - lower[1]) / edges[1]) * vy;
496     }
497   }
498   PetscCall(VecRestoreArray(coordinates, &coords));
499   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
500   PetscCall(VecDestroy(&coordinates));
501   PetscFunctionReturn(0);
502 }
503 
504 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
505 {
506   PetscInt     vertices[3], numVertices;
507   PetscInt     numFaces       = 2 * faces[0] * faces[1] + 2 * faces[1] * faces[2] + 2 * faces[0] * faces[2];
508   PetscInt     markerTop      = 1;
509   PetscInt     markerBottom   = 1;
510   PetscInt     markerFront    = 1;
511   PetscInt     markerBack     = 1;
512   PetscInt     markerRight    = 1;
513   PetscInt     markerLeft     = 1;
514   PetscBool    markerSeparate = PETSC_FALSE;
515   Vec          coordinates;
516   PetscSection coordSection;
517   PetscScalar *coords;
518   PetscInt     coordSize;
519   PetscMPIInt  rank;
520   PetscInt     v, vx, vy, vz;
521   PetscInt     voffset, iface = 0, cone[4];
522 
523   PetscFunctionBegin;
524   PetscCheck(faces[0] >= 1 && faces[1] >= 1 && faces[2] >= 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
525   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
526   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
527   if (markerSeparate) {
528     markerBottom = 1;
529     markerTop    = 2;
530     markerFront  = 3;
531     markerBack   = 4;
532     markerRight  = 5;
533     markerLeft   = 6;
534   }
535   vertices[0] = faces[0] + 1;
536   vertices[1] = faces[1] + 1;
537   vertices[2] = faces[2] + 1;
538   numVertices = vertices[0] * vertices[1] * vertices[2];
539   if (rank == 0) {
540     PetscInt f;
541 
542     PetscCall(DMPlexSetChart(dm, 0, numFaces + numVertices));
543     for (f = 0; f < numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
544     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
545 
546     /* Side 0 (Top) */
547     for (vy = 0; vy < faces[1]; vy++) {
548       for (vx = 0; vx < faces[0]; vx++) {
549         voffset = numFaces + vertices[0] * vertices[1] * (vertices[2] - 1) + vy * vertices[0] + vx;
550         cone[0] = voffset;
551         cone[1] = voffset + 1;
552         cone[2] = voffset + vertices[0] + 1;
553         cone[3] = voffset + vertices[0];
554         PetscCall(DMPlexSetCone(dm, iface, cone));
555         PetscCall(DMSetLabelValue(dm, "marker", iface, markerTop));
556         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerTop));
557         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerTop));
558         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerTop));
559         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerTop));
560         iface++;
561       }
562     }
563 
564     /* Side 1 (Bottom) */
565     for (vy = 0; vy < faces[1]; vy++) {
566       for (vx = 0; vx < faces[0]; vx++) {
567         voffset = numFaces + vy * (faces[0] + 1) + vx;
568         cone[0] = voffset + 1;
569         cone[1] = voffset;
570         cone[2] = voffset + vertices[0];
571         cone[3] = voffset + vertices[0] + 1;
572         PetscCall(DMPlexSetCone(dm, iface, cone));
573         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBottom));
574         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBottom));
575         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBottom));
576         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerBottom));
577         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerBottom));
578         iface++;
579       }
580     }
581 
582     /* Side 2 (Front) */
583     for (vz = 0; vz < faces[2]; vz++) {
584       for (vx = 0; vx < faces[0]; vx++) {
585         voffset = numFaces + vz * vertices[0] * vertices[1] + vx;
586         cone[0] = voffset;
587         cone[1] = voffset + 1;
588         cone[2] = voffset + vertices[0] * vertices[1] + 1;
589         cone[3] = voffset + vertices[0] * vertices[1];
590         PetscCall(DMPlexSetCone(dm, iface, cone));
591         PetscCall(DMSetLabelValue(dm, "marker", iface, markerFront));
592         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerFront));
593         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerFront));
594         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerFront));
595         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerFront));
596         iface++;
597       }
598     }
599 
600     /* Side 3 (Back) */
601     for (vz = 0; vz < faces[2]; vz++) {
602       for (vx = 0; vx < faces[0]; vx++) {
603         voffset = numFaces + vz * vertices[0] * vertices[1] + vertices[0] * (vertices[1] - 1) + vx;
604         cone[0] = voffset + vertices[0] * vertices[1];
605         cone[1] = voffset + vertices[0] * vertices[1] + 1;
606         cone[2] = voffset + 1;
607         cone[3] = voffset;
608         PetscCall(DMPlexSetCone(dm, iface, cone));
609         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBack));
610         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBack));
611         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBack));
612         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerBack));
613         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerBack));
614         iface++;
615       }
616     }
617 
618     /* Side 4 (Left) */
619     for (vz = 0; vz < faces[2]; vz++) {
620       for (vy = 0; vy < faces[1]; vy++) {
621         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0];
622         cone[0] = voffset;
623         cone[1] = voffset + vertices[0] * vertices[1];
624         cone[2] = voffset + vertices[0] * vertices[1] + vertices[0];
625         cone[3] = voffset + vertices[0];
626         PetscCall(DMPlexSetCone(dm, iface, cone));
627         PetscCall(DMSetLabelValue(dm, "marker", iface, markerLeft));
628         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerLeft));
629         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerLeft));
630         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[1] + 0, markerLeft));
631         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerLeft));
632         iface++;
633       }
634     }
635 
636     /* Side 5 (Right) */
637     for (vz = 0; vz < faces[2]; vz++) {
638       for (vy = 0; vy < faces[1]; vy++) {
639         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0] + faces[0];
640         cone[0] = voffset + vertices[0] * vertices[1];
641         cone[1] = voffset;
642         cone[2] = voffset + vertices[0];
643         cone[3] = voffset + vertices[0] * vertices[1] + vertices[0];
644         PetscCall(DMPlexSetCone(dm, iface, cone));
645         PetscCall(DMSetLabelValue(dm, "marker", iface, markerRight));
646         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerRight));
647         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerRight));
648         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerRight));
649         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerRight));
650         iface++;
651       }
652     }
653   }
654   PetscCall(DMPlexSymmetrize(dm));
655   PetscCall(DMPlexStratify(dm));
656   /* Build coordinates */
657   PetscCall(DMSetCoordinateDim(dm, 3));
658   PetscCall(DMGetCoordinateSection(dm, &coordSection));
659   PetscCall(PetscSectionSetNumFields(coordSection, 1));
660   PetscCall(PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices));
661   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 3));
662   for (v = numFaces; v < numFaces + numVertices; ++v) {
663     PetscCall(PetscSectionSetDof(coordSection, v, 3));
664     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 3));
665   }
666   PetscCall(PetscSectionSetUp(coordSection));
667   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
668   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
669   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
670   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
671   PetscCall(VecSetBlockSize(coordinates, 3));
672   PetscCall(VecSetType(coordinates, VECSTANDARD));
673   PetscCall(VecGetArray(coordinates, &coords));
674   for (vz = 0; vz <= faces[2]; ++vz) {
675     for (vy = 0; vy <= faces[1]; ++vy) {
676       for (vx = 0; vx <= faces[0]; ++vx) {
677         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 0] = lower[0] + ((upper[0] - lower[0]) / faces[0]) * vx;
678         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 1] = lower[1] + ((upper[1] - lower[1]) / faces[1]) * vy;
679         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 2] = lower[2] + ((upper[2] - lower[2]) / faces[2]) * vz;
680       }
681     }
682   }
683   PetscCall(VecRestoreArray(coordinates, &coords));
684   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
685   PetscCall(VecDestroy(&coordinates));
686   PetscFunctionReturn(0);
687 }
688 
689 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate)
690 {
691   PetscFunctionBegin;
692   PetscValidLogicalCollectiveInt(dm, dim, 2);
693   PetscCall(DMSetDimension(dm, dim - 1));
694   PetscCall(DMSetCoordinateDim(dm, dim));
695   switch (dim) {
696   case 1:
697     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(dm, lower, upper, faces));
698     break;
699   case 2:
700     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(dm, lower, upper, faces));
701     break;
702   case 3:
703     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(dm, lower, upper, faces));
704     break;
705   default:
706     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Dimension not supported: %" PetscInt_FMT, dim);
707   }
708   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
709   PetscFunctionReturn(0);
710 }
711 
712 /*@C
713   DMPlexCreateBoxSurfaceMesh - Creates a mesh on the surface of the tensor product of unit intervals (box) using tensor cells (hexahedra).
714 
715   Collective
716 
717   Input Parameters:
718 + comm        - The communicator for the `DM` object
719 . dim         - The spatial dimension of the box, so the resulting mesh is has dimension dim-1
720 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
721 . lower       - The lower left corner, or NULL for (0, 0, 0)
722 . upper       - The upper right corner, or NULL for (1, 1, 1)
723 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
724 
725   Output Parameter:
726 . dm  - The `DM` object
727 
728   Level: beginner
729 
730 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateBoxMesh()`, `DMPlexCreateFromFile()`, `DMSetType()`, `DMCreate()`
731 @*/
732 PetscErrorCode DMPlexCreateBoxSurfaceMesh(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate, DM *dm)
733 {
734   PetscInt  fac[3] = {1, 1, 1};
735   PetscReal low[3] = {0, 0, 0};
736   PetscReal upp[3] = {1, 1, 1};
737 
738   PetscFunctionBegin;
739   PetscCall(DMCreate(comm, dm));
740   PetscCall(DMSetType(*dm, DMPLEX));
741   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(*dm, dim, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, interpolate));
742   PetscFunctionReturn(0);
743 }
744 
745 static PetscErrorCode DMPlexCreateLineMesh_Internal(DM dm, PetscInt segments, PetscReal lower, PetscReal upper, DMBoundaryType bd)
746 {
747   PetscInt     i, fStart, fEnd, numCells = 0, numVerts = 0;
748   PetscInt     numPoints[2], *coneSize, *cones, *coneOrientations;
749   PetscScalar *vertexCoords;
750   PetscReal    L, maxCell;
751   PetscBool    markerSeparate = PETSC_FALSE;
752   PetscInt     markerLeft = 1, faceMarkerLeft = 1;
753   PetscInt     markerRight = 1, faceMarkerRight = 2;
754   PetscBool    wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
755   PetscMPIInt  rank;
756 
757   PetscFunctionBegin;
758   PetscValidPointer(dm, 1);
759 
760   PetscCall(DMSetDimension(dm, 1));
761   PetscCall(DMCreateLabel(dm, "marker"));
762   PetscCall(DMCreateLabel(dm, "Face Sets"));
763 
764   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
765   if (rank == 0) numCells = segments;
766   if (rank == 0) numVerts = segments + (wrap ? 0 : 1);
767 
768   numPoints[0] = numVerts;
769   numPoints[1] = numCells;
770   PetscCall(PetscMalloc4(numCells + numVerts, &coneSize, numCells * 2, &cones, numCells + numVerts, &coneOrientations, numVerts, &vertexCoords));
771   PetscCall(PetscArrayzero(coneOrientations, numCells + numVerts));
772   for (i = 0; i < numCells; ++i) coneSize[i] = 2;
773   for (i = 0; i < numVerts; ++i) coneSize[numCells + i] = 0;
774   for (i = 0; i < numCells; ++i) {
775     cones[2 * i]     = numCells + i % numVerts;
776     cones[2 * i + 1] = numCells + (i + 1) % numVerts;
777   }
778   for (i = 0; i < numVerts; ++i) vertexCoords[i] = lower + (upper - lower) * ((PetscReal)i / (PetscReal)numCells);
779   PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
780   PetscCall(PetscFree4(coneSize, cones, coneOrientations, vertexCoords));
781 
782   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
783   if (markerSeparate) {
784     markerLeft  = faceMarkerLeft;
785     markerRight = faceMarkerRight;
786   }
787   if (!wrap && rank == 0) {
788     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
789     PetscCall(DMSetLabelValue(dm, "marker", fStart, markerLeft));
790     PetscCall(DMSetLabelValue(dm, "marker", fEnd - 1, markerRight));
791     PetscCall(DMSetLabelValue(dm, "Face Sets", fStart, faceMarkerLeft));
792     PetscCall(DMSetLabelValue(dm, "Face Sets", fEnd - 1, faceMarkerRight));
793   }
794   if (wrap) {
795     L       = upper - lower;
796     maxCell = (PetscReal)1.1 * (L / (PetscReal)PetscMax(1, segments));
797     PetscCall(DMSetPeriodicity(dm, &maxCell, &lower, &L));
798   }
799   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
800   PetscFunctionReturn(0);
801 }
802 
803 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
804 {
805   DM      boundary, vol;
806   DMLabel bdlabel;
807 
808   PetscFunctionBegin;
809   PetscValidPointer(dm, 1);
810   for (PetscInt i = 0; i < dim; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
811   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &boundary));
812   PetscCall(DMSetType(boundary, DMPLEX));
813   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(boundary, dim, faces, lower, upper, PETSC_FALSE));
814   PetscCall(DMPlexGenerate(boundary, NULL, interpolate, &vol));
815   PetscCall(DMGetLabel(vol, "marker", &bdlabel));
816   if (bdlabel) PetscCall(DMPlexLabelComplete(vol, bdlabel));
817   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_FALSE, vol));
818   PetscCall(DMPlexReplace_Internal(dm, &vol));
819   PetscCall(DMDestroy(&boundary));
820   PetscFunctionReturn(0);
821 }
822 
823 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
824 {
825   DMLabel     cutLabel  = NULL;
826   PetscInt    markerTop = 1, faceMarkerTop = 1;
827   PetscInt    markerBottom = 1, faceMarkerBottom = 1;
828   PetscInt    markerFront = 1, faceMarkerFront = 1;
829   PetscInt    markerBack = 1, faceMarkerBack = 1;
830   PetscInt    markerRight = 1, faceMarkerRight = 1;
831   PetscInt    markerLeft = 1, faceMarkerLeft = 1;
832   PetscInt    dim;
833   PetscBool   markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
834   PetscMPIInt rank;
835 
836   PetscFunctionBegin;
837   PetscCall(DMGetDimension(dm, &dim));
838   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
839   PetscCall(DMCreateLabel(dm, "marker"));
840   PetscCall(DMCreateLabel(dm, "Face Sets"));
841   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
842   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST || bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST || bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
843     if (cutMarker) {
844       PetscCall(DMCreateLabel(dm, "periodic_cut"));
845       PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
846     }
847   }
848   switch (dim) {
849   case 2:
850     faceMarkerTop    = 3;
851     faceMarkerBottom = 1;
852     faceMarkerRight  = 2;
853     faceMarkerLeft   = 4;
854     break;
855   case 3:
856     faceMarkerBottom = 1;
857     faceMarkerTop    = 2;
858     faceMarkerFront  = 3;
859     faceMarkerBack   = 4;
860     faceMarkerRight  = 5;
861     faceMarkerLeft   = 6;
862     break;
863   default:
864     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
865   }
866   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
867   if (markerSeparate) {
868     markerBottom = faceMarkerBottom;
869     markerTop    = faceMarkerTop;
870     markerFront  = faceMarkerFront;
871     markerBack   = faceMarkerBack;
872     markerRight  = faceMarkerRight;
873     markerLeft   = faceMarkerLeft;
874   }
875   {
876     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
877     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
878     const PetscInt numZEdges    = rank == 0 ? edges[2] : 0;
879     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0] + 1) : 0;
880     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1] + 1) : 0;
881     const PetscInt numZVertices = rank == 0 ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2] + 1) : 0;
882     const PetscInt numCells     = numXEdges * numYEdges * numZEdges;
883     const PetscInt numXFaces    = numYEdges * numZEdges;
884     const PetscInt numYFaces    = numXEdges * numZEdges;
885     const PetscInt numZFaces    = numXEdges * numYEdges;
886     const PetscInt numTotXFaces = numXVertices * numXFaces;
887     const PetscInt numTotYFaces = numYVertices * numYFaces;
888     const PetscInt numTotZFaces = numZVertices * numZFaces;
889     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
890     const PetscInt numTotXEdges = numXEdges * numYVertices * numZVertices;
891     const PetscInt numTotYEdges = numYEdges * numXVertices * numZVertices;
892     const PetscInt numTotZEdges = numZEdges * numXVertices * numYVertices;
893     const PetscInt numVertices  = numXVertices * numYVertices * numZVertices;
894     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
895     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
896     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
897     const PetscInt firstYFace   = firstXFace + numTotXFaces;
898     const PetscInt firstZFace   = firstYFace + numTotYFaces;
899     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
900     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
901     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
902     Vec            coordinates;
903     PetscSection   coordSection;
904     PetscScalar   *coords;
905     PetscInt       coordSize;
906     PetscInt       v, vx, vy, vz;
907     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
908 
909     PetscCall(DMPlexSetChart(dm, 0, numCells + numFaces + numEdges + numVertices));
910     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
911     for (f = firstXFace; f < firstXFace + numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
912     for (e = firstXEdge; e < firstXEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
913     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
914     /* Build cells */
915     for (fz = 0; fz < numZEdges; ++fz) {
916       for (fy = 0; fy < numYEdges; ++fy) {
917         for (fx = 0; fx < numXEdges; ++fx) {
918           PetscInt cell  = (fz * numYEdges + fy) * numXEdges + fx;
919           PetscInt faceB = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
920           PetscInt faceT = firstZFace + (fy * numXEdges + fx) * numZVertices + ((fz + 1) % numZVertices);
921           PetscInt faceF = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
922           PetscInt faceK = firstYFace + (fz * numXEdges + fx) * numYVertices + ((fy + 1) % numYVertices);
923           PetscInt faceL = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
924           PetscInt faceR = firstXFace + (fz * numYEdges + fy) * numXVertices + ((fx + 1) % numXVertices);
925           /* B,  T,  F,  K,  R,  L */
926           PetscInt ornt[6] = {-2, 0, 0, -3, 0, -2}; /* ??? */
927           PetscInt cone[6];
928 
929           /* no boundary twisting in 3D */
930           cone[0] = faceB;
931           cone[1] = faceT;
932           cone[2] = faceF;
933           cone[3] = faceK;
934           cone[4] = faceR;
935           cone[5] = faceL;
936           PetscCall(DMPlexSetCone(dm, cell, cone));
937           PetscCall(DMPlexSetConeOrientation(dm, cell, ornt));
938           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
939           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
940           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
941         }
942       }
943     }
944     /* Build x faces */
945     for (fz = 0; fz < numZEdges; ++fz) {
946       for (fy = 0; fy < numYEdges; ++fy) {
947         for (fx = 0; fx < numXVertices; ++fx) {
948           PetscInt face    = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
949           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
950           PetscInt edgeR   = firstZEdge + (((fy + 1) % numYVertices) * numXVertices + fx) * numZEdges + fz;
951           PetscInt edgeB   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
952           PetscInt edgeT   = firstYEdge + (((fz + 1) % numZVertices) * numXVertices + fx) * numYEdges + fy;
953           PetscInt ornt[4] = {0, 0, -1, -1};
954           PetscInt cone[4];
955 
956           if (dim == 3) {
957             /* markers */
958             if (bdX != DM_BOUNDARY_PERIODIC) {
959               if (fx == numXVertices - 1) {
960                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight));
961                 PetscCall(DMSetLabelValue(dm, "marker", face, markerRight));
962               } else if (fx == 0) {
963                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft));
964                 PetscCall(DMSetLabelValue(dm, "marker", face, markerLeft));
965               }
966             }
967           }
968           cone[0] = edgeB;
969           cone[1] = edgeR;
970           cone[2] = edgeT;
971           cone[3] = edgeL;
972           PetscCall(DMPlexSetCone(dm, face, cone));
973           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
974         }
975       }
976     }
977     /* Build y faces */
978     for (fz = 0; fz < numZEdges; ++fz) {
979       for (fx = 0; fx < numXEdges; ++fx) {
980         for (fy = 0; fy < numYVertices; ++fy) {
981           PetscInt face    = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
982           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
983           PetscInt edgeR   = firstZEdge + (fy * numXVertices + ((fx + 1) % numXVertices)) * numZEdges + fz;
984           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
985           PetscInt edgeT   = firstXEdge + (((fz + 1) % numZVertices) * numYVertices + fy) * numXEdges + fx;
986           PetscInt ornt[4] = {0, 0, -1, -1};
987           PetscInt cone[4];
988 
989           if (dim == 3) {
990             /* markers */
991             if (bdY != DM_BOUNDARY_PERIODIC) {
992               if (fy == numYVertices - 1) {
993                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack));
994                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBack));
995               } else if (fy == 0) {
996                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront));
997                 PetscCall(DMSetLabelValue(dm, "marker", face, markerFront));
998               }
999             }
1000           }
1001           cone[0] = edgeB;
1002           cone[1] = edgeR;
1003           cone[2] = edgeT;
1004           cone[3] = edgeL;
1005           PetscCall(DMPlexSetCone(dm, face, cone));
1006           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1007         }
1008       }
1009     }
1010     /* Build z faces */
1011     for (fy = 0; fy < numYEdges; ++fy) {
1012       for (fx = 0; fx < numXEdges; ++fx) {
1013         for (fz = 0; fz < numZVertices; fz++) {
1014           PetscInt face    = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
1015           PetscInt edgeL   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
1016           PetscInt edgeR   = firstYEdge + (fz * numXVertices + ((fx + 1) % numXVertices)) * numYEdges + fy;
1017           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1018           PetscInt edgeT   = firstXEdge + (fz * numYVertices + ((fy + 1) % numYVertices)) * numXEdges + fx;
1019           PetscInt ornt[4] = {0, 0, -1, -1};
1020           PetscInt cone[4];
1021 
1022           if (dim == 2) {
1023             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges - 1) {
1024               edgeR += numYEdges - 1 - 2 * fy;
1025               ornt[1] = -1;
1026             }
1027             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges - 1) {
1028               edgeT += numXEdges - 1 - 2 * fx;
1029               ornt[2] = 0;
1030             }
1031             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1032             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1033           } else {
1034             /* markers */
1035             if (bdZ != DM_BOUNDARY_PERIODIC) {
1036               if (fz == numZVertices - 1) {
1037                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop));
1038                 PetscCall(DMSetLabelValue(dm, "marker", face, markerTop));
1039               } else if (fz == 0) {
1040                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom));
1041                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBottom));
1042               }
1043             }
1044           }
1045           cone[0] = edgeB;
1046           cone[1] = edgeR;
1047           cone[2] = edgeT;
1048           cone[3] = edgeL;
1049           PetscCall(DMPlexSetCone(dm, face, cone));
1050           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1051         }
1052       }
1053     }
1054     /* Build Z edges*/
1055     for (vy = 0; vy < numYVertices; vy++) {
1056       for (vx = 0; vx < numXVertices; vx++) {
1057         for (ez = 0; ez < numZEdges; ez++) {
1058           const PetscInt edge    = firstZEdge + (vy * numXVertices + vx) * numZEdges + ez;
1059           const PetscInt vertexB = firstVertex + (ez * numYVertices + vy) * numXVertices + vx;
1060           const PetscInt vertexT = firstVertex + (((ez + 1) % numZVertices) * numYVertices + vy) * numXVertices + vx;
1061           PetscInt       cone[2];
1062 
1063           cone[0] = vertexB;
1064           cone[1] = vertexT;
1065           PetscCall(DMPlexSetCone(dm, edge, cone));
1066           if (dim == 3) {
1067             if (bdX != DM_BOUNDARY_PERIODIC) {
1068               if (vx == numXVertices - 1) {
1069                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1070                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1071                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1072               } else if (vx == 0) {
1073                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1074                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1075                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1076               }
1077             }
1078             if (bdY != DM_BOUNDARY_PERIODIC) {
1079               if (vy == numYVertices - 1) {
1080                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1081                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1082                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1083               } else if (vy == 0) {
1084                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1085                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1086                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1087               }
1088             }
1089           }
1090         }
1091       }
1092     }
1093     /* Build Y edges*/
1094     for (vz = 0; vz < numZVertices; vz++) {
1095       for (vx = 0; vx < numXVertices; vx++) {
1096         for (ey = 0; ey < numYEdges; ey++) {
1097           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges - 1) ? (numXVertices - vx - 1) : (vz * numYVertices + ((ey + 1) % numYVertices)) * numXVertices + vx;
1098           const PetscInt edge    = firstYEdge + (vz * numXVertices + vx) * numYEdges + ey;
1099           const PetscInt vertexF = firstVertex + (vz * numYVertices + ey) * numXVertices + vx;
1100           const PetscInt vertexK = firstVertex + nextv;
1101           PetscInt       cone[2];
1102 
1103           cone[0] = vertexF;
1104           cone[1] = vertexK;
1105           PetscCall(DMPlexSetCone(dm, edge, cone));
1106           if (dim == 2) {
1107             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1108               if (vx == numXVertices - 1) {
1109                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight));
1110                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1111                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1112                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1113               } else if (vx == 0) {
1114                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft));
1115                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1116                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1117                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1118               }
1119             } else {
1120               if (vx == 0 && cutLabel) {
1121                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1122                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1123                 if (ey == numYEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1124               }
1125             }
1126           } else {
1127             if (bdX != DM_BOUNDARY_PERIODIC) {
1128               if (vx == numXVertices - 1) {
1129                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1130                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1131                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1132               } else if (vx == 0) {
1133                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1134                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1135                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1136               }
1137             }
1138             if (bdZ != DM_BOUNDARY_PERIODIC) {
1139               if (vz == numZVertices - 1) {
1140                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1141                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1142                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1143               } else if (vz == 0) {
1144                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1145                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1146                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1147               }
1148             }
1149           }
1150         }
1151       }
1152     }
1153     /* Build X edges*/
1154     for (vz = 0; vz < numZVertices; vz++) {
1155       for (vy = 0; vy < numYVertices; vy++) {
1156         for (ex = 0; ex < numXEdges; ex++) {
1157           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges - 1) ? (numYVertices - vy - 1) * numXVertices : (vz * numYVertices + vy) * numXVertices + (ex + 1) % numXVertices;
1158           const PetscInt edge    = firstXEdge + (vz * numYVertices + vy) * numXEdges + ex;
1159           const PetscInt vertexL = firstVertex + (vz * numYVertices + vy) * numXVertices + ex;
1160           const PetscInt vertexR = firstVertex + nextv;
1161           PetscInt       cone[2];
1162 
1163           cone[0] = vertexL;
1164           cone[1] = vertexR;
1165           PetscCall(DMPlexSetCone(dm, edge, cone));
1166           if (dim == 2) {
1167             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1168               if (vy == numYVertices - 1) {
1169                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop));
1170                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1171                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1172                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1173               } else if (vy == 0) {
1174                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom));
1175                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1176                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1177                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1178               }
1179             } else {
1180               if (vy == 0 && cutLabel) {
1181                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1182                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1183                 if (ex == numXEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1184               }
1185             }
1186           } else {
1187             if (bdY != DM_BOUNDARY_PERIODIC) {
1188               if (vy == numYVertices - 1) {
1189                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1190                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1191                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1192               } else if (vy == 0) {
1193                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1194                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1195                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1196               }
1197             }
1198             if (bdZ != DM_BOUNDARY_PERIODIC) {
1199               if (vz == numZVertices - 1) {
1200                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1201                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1202                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1203               } else if (vz == 0) {
1204                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1205                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1206                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1207               }
1208             }
1209           }
1210         }
1211       }
1212     }
1213     PetscCall(DMPlexSymmetrize(dm));
1214     PetscCall(DMPlexStratify(dm));
1215     /* Build coordinates */
1216     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1217     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1218     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1219     PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVertices));
1220     for (v = firstVertex; v < firstVertex + numVertices; ++v) {
1221       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1222       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1223     }
1224     PetscCall(PetscSectionSetUp(coordSection));
1225     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1226     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1227     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1228     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1229     PetscCall(VecSetBlockSize(coordinates, dim));
1230     PetscCall(VecSetType(coordinates, VECSTANDARD));
1231     PetscCall(VecGetArray(coordinates, &coords));
1232     for (vz = 0; vz < numZVertices; ++vz) {
1233       for (vy = 0; vy < numYVertices; ++vy) {
1234         for (vx = 0; vx < numXVertices; ++vx) {
1235           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * vx;
1236           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * vy;
1237           if (dim == 3) coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 2] = lower[2] + ((upper[2] - lower[2]) / numZEdges) * vz;
1238         }
1239       }
1240     }
1241     PetscCall(VecRestoreArray(coordinates, &coords));
1242     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1243     PetscCall(VecDestroy(&coordinates));
1244   }
1245   PetscFunctionReturn(0);
1246 }
1247 
1248 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1249 {
1250   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1251   PetscInt       fac[3] = {0, 0, 0}, d;
1252 
1253   PetscFunctionBegin;
1254   PetscValidPointer(dm, 1);
1255   PetscValidLogicalCollectiveInt(dm, dim, 2);
1256   PetscCall(DMSetDimension(dm, dim));
1257   for (d = 0; d < dim; ++d) {
1258     fac[d] = faces[d];
1259     bdt[d] = periodicity[d];
1260   }
1261   PetscCall(DMPlexCreateCubeMesh_Internal(dm, lower, upper, fac, bdt[0], bdt[1], bdt[2]));
1262   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST || periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST || (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
1263     PetscReal L[3]       = {-1., -1., 0.};
1264     PetscReal maxCell[3] = {-1., -1., 0.};
1265 
1266     for (d = 0; d < dim; ++d) {
1267       if (periodicity[d] != DM_BOUNDARY_NONE) {
1268         L[d]       = upper[d] - lower[d];
1269         maxCell[d] = 1.1 * (L[d] / PetscMax(1, faces[d]));
1270       }
1271     }
1272     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1273   }
1274   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
1275   PetscFunctionReturn(0);
1276 }
1277 
1278 static PetscErrorCode DMPlexCreateBoxMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1279 {
1280   PetscFunctionBegin;
1281   if (dim == 1) PetscCall(DMPlexCreateLineMesh_Internal(dm, faces[0], lower[0], upper[0], periodicity[0]));
1282   else if (simplex) PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1283   else PetscCall(DMPlexCreateBoxMesh_Tensor_Internal(dm, dim, faces, lower, upper, periodicity));
1284   if (!interpolate && dim > 1 && !simplex) {
1285     DM udm;
1286 
1287     PetscCall(DMPlexUninterpolate(dm, &udm));
1288     PetscCall(DMPlexCopyCoordinates(dm, udm));
1289     PetscCall(DMPlexReplace_Internal(dm, &udm));
1290   }
1291   PetscFunctionReturn(0);
1292 }
1293 
1294 /*@C
1295   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
1296 
1297   Collective
1298 
1299   Input Parameters:
1300 + comm        - The communicator for the `DM` object
1301 . dim         - The spatial dimension
1302 . simplex     - `PETSC_TRUE` for simplices, `PETSC_FALSE` for tensor cells
1303 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1304 . lower       - The lower left corner, or NULL for (0, 0, 0)
1305 . upper       - The upper right corner, or NULL for (1, 1, 1)
1306 . periodicity - The boundary type for the X,Y,Z direction, or NULL for `DM_BOUNDARY_NONE`
1307 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1308 
1309   Output Parameter:
1310 . dm  - The `DM` object
1311 
1312   Level: beginner
1313 
1314   Note:
1315    To customize this mesh using options, use
1316 .vb
1317   DMCreate(comm, &dm);
1318   DMSetType(dm, DMPLEX);
1319   DMSetFromOptions(dm);
1320 .ve
1321 and use the options in `DMSetFromOptions()`.
1322 
1323   Here is the numbering returned for 2 faces in each direction for tensor cells:
1324 .vb
1325  10---17---11---18----12
1326   |         |         |
1327   |         |         |
1328  20    2   22    3    24
1329   |         |         |
1330   |         |         |
1331   7---15----8---16----9
1332   |         |         |
1333   |         |         |
1334  19    0   21    1   23
1335   |         |         |
1336   |         |         |
1337   4---13----5---14----6
1338 .ve
1339 and for simplicial cells
1340 .vb
1341  14----8---15----9----16
1342   |\     5  |\      7 |
1343   | \       | \       |
1344  13   2    14    3    15
1345   | 4   \   | 6   \   |
1346   |       \ |       \ |
1347  11----6---12----7----13
1348   |\        |\        |
1349   | \    1  | \     3 |
1350  10   0    11    1    12
1351   | 0   \   | 2   \   |
1352   |       \ |       \ |
1353   8----4----9----5----10
1354 .ve
1355 
1356 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
1357 @*/
1358 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
1359 {
1360   PetscInt       fac[3] = {1, 1, 1};
1361   PetscReal      low[3] = {0, 0, 0};
1362   PetscReal      upp[3] = {1, 1, 1};
1363   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1364 
1365   PetscFunctionBegin;
1366   PetscCall(DMCreate(comm, dm));
1367   PetscCall(DMSetType(*dm, DMPLEX));
1368   PetscCall(DMPlexCreateBoxMesh_Internal(*dm, dim, simplex, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt, interpolate));
1369   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1370   PetscFunctionReturn(0);
1371 }
1372 
1373 static PetscErrorCode DMPlexCreateWedgeBoxMesh_Internal(DM dm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1374 {
1375   DM       bdm, vol;
1376   PetscInt i;
1377 
1378   PetscFunctionBegin;
1379   for (i = 0; i < 3; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity not yet supported");
1380   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &bdm));
1381   PetscCall(DMSetType(bdm, DMPLEX));
1382   PetscCall(DMSetDimension(bdm, 2));
1383   PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(bdm, 2, faces, lower, upper, periodicity, PETSC_TRUE));
1384   PetscCall(DMPlexExtrude(bdm, faces[2], upper[2] - lower[2], PETSC_TRUE, PETSC_FALSE, NULL, NULL, &vol));
1385   PetscCall(DMDestroy(&bdm));
1386   PetscCall(DMPlexReplace_Internal(dm, &vol));
1387   if (lower[2] != 0.0) {
1388     Vec          v;
1389     PetscScalar *x;
1390     PetscInt     cDim, n;
1391 
1392     PetscCall(DMGetCoordinatesLocal(dm, &v));
1393     PetscCall(VecGetBlockSize(v, &cDim));
1394     PetscCall(VecGetLocalSize(v, &n));
1395     PetscCall(VecGetArray(v, &x));
1396     x += cDim;
1397     for (i = 0; i < n; i += cDim) x[i] += lower[2];
1398     PetscCall(VecRestoreArray(v, &x));
1399     PetscCall(DMSetCoordinatesLocal(dm, v));
1400   }
1401   PetscFunctionReturn(0);
1402 }
1403 
1404 /*@
1405   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tesselating the (x,y) plane and extruding in the third direction using wedge cells.
1406 
1407   Collective
1408 
1409   Input Parameters:
1410 + comm        - The communicator for the `DM` object
1411 . faces       - Number of faces per dimension, or NULL for (1, 1, 1)
1412 . lower       - The lower left corner, or NULL for (0, 0, 0)
1413 . upper       - The upper right corner, or NULL for (1, 1, 1)
1414 . periodicity - The boundary type for the X,Y,Z direction, or NULL for `DM_BOUNDARY_NONE`
1415 . orderHeight - If `PETSC_TRUE`, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1416 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1417 
1418   Output Parameter:
1419 . dm  - The `DM` object
1420 
1421   Level: beginner
1422 
1423 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateWedgeCylinderMesh()`, `DMExtrude()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1424 @*/
1425 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool orderHeight, PetscBool interpolate, DM *dm)
1426 {
1427   PetscInt       fac[3] = {1, 1, 1};
1428   PetscReal      low[3] = {0, 0, 0};
1429   PetscReal      upp[3] = {1, 1, 1};
1430   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1431 
1432   PetscFunctionBegin;
1433   PetscCall(DMCreate(comm, dm));
1434   PetscCall(DMSetType(*dm, DMPLEX));
1435   PetscCall(DMPlexCreateWedgeBoxMesh_Internal(*dm, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt));
1436   if (!interpolate) {
1437     DM udm;
1438 
1439     PetscCall(DMPlexUninterpolate(*dm, &udm));
1440     PetscCall(DMPlexReplace_Internal(*dm, &udm));
1441   }
1442   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1443   PetscFunctionReturn(0);
1444 }
1445 
1446 /*@C
1447   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all `DM` options in the database.
1448 
1449   Logically Collective on dm
1450 
1451   Input Parameters:
1452 + dm - the DM context
1453 - prefix - the prefix to prepend to all option names
1454 
1455   Level: advanced
1456 
1457   Note:
1458   A hyphen (-) must NOT be given at the beginning of the prefix name.
1459   The first character of all runtime options is AUTOMATICALLY the hyphen.
1460 
1461 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `SNESSetFromOptions()`
1462 @*/
1463 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
1464 {
1465   DM_Plex *mesh = (DM_Plex *)dm->data;
1466 
1467   PetscFunctionBegin;
1468   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1469   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, prefix));
1470   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)mesh->partitioner, prefix));
1471   PetscFunctionReturn(0);
1472 }
1473 
1474 /* Remap geometry to cylinder
1475    TODO: This only works for a single refinement, then it is broken
1476 
1477      Interior square: Linear interpolation is correct
1478      The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
1479      such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
1480 
1481        phi     = arctan(y/x)
1482        d_close = sqrt(1/8 + 1/4 sin^2(phi))
1483        d_far   = sqrt(1/2 + sin^2(phi))
1484 
1485      so we remap them using
1486 
1487        x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
1488        y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
1489 
1490      If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
1491 */
1492 static void snapToCylinder(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
1493 {
1494   const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1495   const PetscReal ds2 = 0.5 * dis;
1496 
1497   if ((PetscAbsScalar(u[0]) <= ds2) && (PetscAbsScalar(u[1]) <= ds2)) {
1498     f0[0] = u[0];
1499     f0[1] = u[1];
1500   } else {
1501     PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
1502 
1503     x    = PetscRealPart(u[0]);
1504     y    = PetscRealPart(u[1]);
1505     phi  = PetscAtan2Real(y, x);
1506     sinp = PetscSinReal(phi);
1507     cosp = PetscCosReal(phi);
1508     if ((PetscAbsReal(phi) > PETSC_PI / 4.0) && (PetscAbsReal(phi) < 3.0 * PETSC_PI / 4.0)) {
1509       dc = PetscAbsReal(ds2 / sinp);
1510       df = PetscAbsReal(dis / sinp);
1511       xc = ds2 * x / PetscAbsReal(y);
1512       yc = ds2 * PetscSignReal(y);
1513     } else {
1514       dc = PetscAbsReal(ds2 / cosp);
1515       df = PetscAbsReal(dis / cosp);
1516       xc = ds2 * PetscSignReal(x);
1517       yc = ds2 * y / PetscAbsReal(x);
1518     }
1519     f0[0] = xc + (u[0] - xc) * (1.0 - dc) / (df - dc);
1520     f0[1] = yc + (u[1] - yc) * (1.0 - dc) / (df - dc);
1521   }
1522   f0[2] = u[2];
1523 }
1524 
1525 static PetscErrorCode DMPlexCreateHexCylinderMesh_Internal(DM dm, DMBoundaryType periodicZ)
1526 {
1527   const PetscInt dim = 3;
1528   PetscInt       numCells, numVertices;
1529   PetscMPIInt    rank;
1530 
1531   PetscFunctionBegin;
1532   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1533   PetscCall(DMSetDimension(dm, dim));
1534   /* Create topology */
1535   {
1536     PetscInt cone[8], c;
1537 
1538     numCells    = rank == 0 ? 5 : 0;
1539     numVertices = rank == 0 ? 16 : 0;
1540     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1541       numCells *= 3;
1542       numVertices = rank == 0 ? 24 : 0;
1543     }
1544     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1545     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 8));
1546     PetscCall(DMSetUp(dm));
1547     if (rank == 0) {
1548       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1549         cone[0] = 15;
1550         cone[1] = 18;
1551         cone[2] = 17;
1552         cone[3] = 16;
1553         cone[4] = 31;
1554         cone[5] = 32;
1555         cone[6] = 33;
1556         cone[7] = 34;
1557         PetscCall(DMPlexSetCone(dm, 0, cone));
1558         cone[0] = 16;
1559         cone[1] = 17;
1560         cone[2] = 24;
1561         cone[3] = 23;
1562         cone[4] = 32;
1563         cone[5] = 36;
1564         cone[6] = 37;
1565         cone[7] = 33; /* 22 25 26 21 */
1566         PetscCall(DMPlexSetCone(dm, 1, cone));
1567         cone[0] = 18;
1568         cone[1] = 27;
1569         cone[2] = 24;
1570         cone[3] = 17;
1571         cone[4] = 34;
1572         cone[5] = 33;
1573         cone[6] = 37;
1574         cone[7] = 38;
1575         PetscCall(DMPlexSetCone(dm, 2, cone));
1576         cone[0] = 29;
1577         cone[1] = 27;
1578         cone[2] = 18;
1579         cone[3] = 15;
1580         cone[4] = 35;
1581         cone[5] = 31;
1582         cone[6] = 34;
1583         cone[7] = 38;
1584         PetscCall(DMPlexSetCone(dm, 3, cone));
1585         cone[0] = 29;
1586         cone[1] = 15;
1587         cone[2] = 16;
1588         cone[3] = 23;
1589         cone[4] = 35;
1590         cone[5] = 36;
1591         cone[6] = 32;
1592         cone[7] = 31;
1593         PetscCall(DMPlexSetCone(dm, 4, cone));
1594 
1595         cone[0] = 31;
1596         cone[1] = 34;
1597         cone[2] = 33;
1598         cone[3] = 32;
1599         cone[4] = 19;
1600         cone[5] = 22;
1601         cone[6] = 21;
1602         cone[7] = 20;
1603         PetscCall(DMPlexSetCone(dm, 5, cone));
1604         cone[0] = 32;
1605         cone[1] = 33;
1606         cone[2] = 37;
1607         cone[3] = 36;
1608         cone[4] = 22;
1609         cone[5] = 25;
1610         cone[6] = 26;
1611         cone[7] = 21;
1612         PetscCall(DMPlexSetCone(dm, 6, cone));
1613         cone[0] = 34;
1614         cone[1] = 38;
1615         cone[2] = 37;
1616         cone[3] = 33;
1617         cone[4] = 20;
1618         cone[5] = 21;
1619         cone[6] = 26;
1620         cone[7] = 28;
1621         PetscCall(DMPlexSetCone(dm, 7, cone));
1622         cone[0] = 35;
1623         cone[1] = 38;
1624         cone[2] = 34;
1625         cone[3] = 31;
1626         cone[4] = 30;
1627         cone[5] = 19;
1628         cone[6] = 20;
1629         cone[7] = 28;
1630         PetscCall(DMPlexSetCone(dm, 8, cone));
1631         cone[0] = 35;
1632         cone[1] = 31;
1633         cone[2] = 32;
1634         cone[3] = 36;
1635         cone[4] = 30;
1636         cone[5] = 25;
1637         cone[6] = 22;
1638         cone[7] = 19;
1639         PetscCall(DMPlexSetCone(dm, 9, cone));
1640 
1641         cone[0] = 19;
1642         cone[1] = 20;
1643         cone[2] = 21;
1644         cone[3] = 22;
1645         cone[4] = 15;
1646         cone[5] = 16;
1647         cone[6] = 17;
1648         cone[7] = 18;
1649         PetscCall(DMPlexSetCone(dm, 10, cone));
1650         cone[0] = 22;
1651         cone[1] = 21;
1652         cone[2] = 26;
1653         cone[3] = 25;
1654         cone[4] = 16;
1655         cone[5] = 23;
1656         cone[6] = 24;
1657         cone[7] = 17;
1658         PetscCall(DMPlexSetCone(dm, 11, cone));
1659         cone[0] = 20;
1660         cone[1] = 28;
1661         cone[2] = 26;
1662         cone[3] = 21;
1663         cone[4] = 18;
1664         cone[5] = 17;
1665         cone[6] = 24;
1666         cone[7] = 27;
1667         PetscCall(DMPlexSetCone(dm, 12, cone));
1668         cone[0] = 30;
1669         cone[1] = 28;
1670         cone[2] = 20;
1671         cone[3] = 19;
1672         cone[4] = 29;
1673         cone[5] = 15;
1674         cone[6] = 18;
1675         cone[7] = 27;
1676         PetscCall(DMPlexSetCone(dm, 13, cone));
1677         cone[0] = 30;
1678         cone[1] = 19;
1679         cone[2] = 22;
1680         cone[3] = 25;
1681         cone[4] = 29;
1682         cone[5] = 23;
1683         cone[6] = 16;
1684         cone[7] = 15;
1685         PetscCall(DMPlexSetCone(dm, 14, cone));
1686       } else {
1687         cone[0] = 5;
1688         cone[1] = 8;
1689         cone[2] = 7;
1690         cone[3] = 6;
1691         cone[4] = 9;
1692         cone[5] = 12;
1693         cone[6] = 11;
1694         cone[7] = 10;
1695         PetscCall(DMPlexSetCone(dm, 0, cone));
1696         cone[0] = 6;
1697         cone[1] = 7;
1698         cone[2] = 14;
1699         cone[3] = 13;
1700         cone[4] = 12;
1701         cone[5] = 15;
1702         cone[6] = 16;
1703         cone[7] = 11;
1704         PetscCall(DMPlexSetCone(dm, 1, cone));
1705         cone[0] = 8;
1706         cone[1] = 17;
1707         cone[2] = 14;
1708         cone[3] = 7;
1709         cone[4] = 10;
1710         cone[5] = 11;
1711         cone[6] = 16;
1712         cone[7] = 18;
1713         PetscCall(DMPlexSetCone(dm, 2, cone));
1714         cone[0] = 19;
1715         cone[1] = 17;
1716         cone[2] = 8;
1717         cone[3] = 5;
1718         cone[4] = 20;
1719         cone[5] = 9;
1720         cone[6] = 10;
1721         cone[7] = 18;
1722         PetscCall(DMPlexSetCone(dm, 3, cone));
1723         cone[0] = 19;
1724         cone[1] = 5;
1725         cone[2] = 6;
1726         cone[3] = 13;
1727         cone[4] = 20;
1728         cone[5] = 15;
1729         cone[6] = 12;
1730         cone[7] = 9;
1731         PetscCall(DMPlexSetCone(dm, 4, cone));
1732       }
1733     }
1734     PetscCall(DMPlexSymmetrize(dm));
1735     PetscCall(DMPlexStratify(dm));
1736   }
1737   /* Create cube geometry */
1738   {
1739     Vec             coordinates;
1740     PetscSection    coordSection;
1741     PetscScalar    *coords;
1742     PetscInt        coordSize, v;
1743     const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1744     const PetscReal ds2 = dis / 2.0;
1745 
1746     /* Build coordinates */
1747     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1748     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1749     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1750     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1751     for (v = numCells; v < numCells + numVertices; ++v) {
1752       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1753       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1754     }
1755     PetscCall(PetscSectionSetUp(coordSection));
1756     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1757     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1758     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1759     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1760     PetscCall(VecSetBlockSize(coordinates, dim));
1761     PetscCall(VecSetType(coordinates, VECSTANDARD));
1762     PetscCall(VecGetArray(coordinates, &coords));
1763     if (rank == 0) {
1764       coords[0 * dim + 0]  = -ds2;
1765       coords[0 * dim + 1]  = -ds2;
1766       coords[0 * dim + 2]  = 0.0;
1767       coords[1 * dim + 0]  = ds2;
1768       coords[1 * dim + 1]  = -ds2;
1769       coords[1 * dim + 2]  = 0.0;
1770       coords[2 * dim + 0]  = ds2;
1771       coords[2 * dim + 1]  = ds2;
1772       coords[2 * dim + 2]  = 0.0;
1773       coords[3 * dim + 0]  = -ds2;
1774       coords[3 * dim + 1]  = ds2;
1775       coords[3 * dim + 2]  = 0.0;
1776       coords[4 * dim + 0]  = -ds2;
1777       coords[4 * dim + 1]  = -ds2;
1778       coords[4 * dim + 2]  = 1.0;
1779       coords[5 * dim + 0]  = -ds2;
1780       coords[5 * dim + 1]  = ds2;
1781       coords[5 * dim + 2]  = 1.0;
1782       coords[6 * dim + 0]  = ds2;
1783       coords[6 * dim + 1]  = ds2;
1784       coords[6 * dim + 2]  = 1.0;
1785       coords[7 * dim + 0]  = ds2;
1786       coords[7 * dim + 1]  = -ds2;
1787       coords[7 * dim + 2]  = 1.0;
1788       coords[8 * dim + 0]  = dis;
1789       coords[8 * dim + 1]  = -dis;
1790       coords[8 * dim + 2]  = 0.0;
1791       coords[9 * dim + 0]  = dis;
1792       coords[9 * dim + 1]  = dis;
1793       coords[9 * dim + 2]  = 0.0;
1794       coords[10 * dim + 0] = dis;
1795       coords[10 * dim + 1] = -dis;
1796       coords[10 * dim + 2] = 1.0;
1797       coords[11 * dim + 0] = dis;
1798       coords[11 * dim + 1] = dis;
1799       coords[11 * dim + 2] = 1.0;
1800       coords[12 * dim + 0] = -dis;
1801       coords[12 * dim + 1] = dis;
1802       coords[12 * dim + 2] = 0.0;
1803       coords[13 * dim + 0] = -dis;
1804       coords[13 * dim + 1] = dis;
1805       coords[13 * dim + 2] = 1.0;
1806       coords[14 * dim + 0] = -dis;
1807       coords[14 * dim + 1] = -dis;
1808       coords[14 * dim + 2] = 0.0;
1809       coords[15 * dim + 0] = -dis;
1810       coords[15 * dim + 1] = -dis;
1811       coords[15 * dim + 2] = 1.0;
1812       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1813         /* 15 31 19 */ coords[16 * dim + 0] = -ds2;
1814         coords[16 * dim + 1]                = -ds2;
1815         coords[16 * dim + 2]                = 0.5;
1816         /* 16 32 22 */ coords[17 * dim + 0] = ds2;
1817         coords[17 * dim + 1]                = -ds2;
1818         coords[17 * dim + 2]                = 0.5;
1819         /* 17 33 21 */ coords[18 * dim + 0] = ds2;
1820         coords[18 * dim + 1]                = ds2;
1821         coords[18 * dim + 2]                = 0.5;
1822         /* 18 34 20 */ coords[19 * dim + 0] = -ds2;
1823         coords[19 * dim + 1]                = ds2;
1824         coords[19 * dim + 2]                = 0.5;
1825         /* 29 35 30 */ coords[20 * dim + 0] = -dis;
1826         coords[20 * dim + 1]                = -dis;
1827         coords[20 * dim + 2]                = 0.5;
1828         /* 23 36 25 */ coords[21 * dim + 0] = dis;
1829         coords[21 * dim + 1]                = -dis;
1830         coords[21 * dim + 2]                = 0.5;
1831         /* 24 37 26 */ coords[22 * dim + 0] = dis;
1832         coords[22 * dim + 1]                = dis;
1833         coords[22 * dim + 2]                = 0.5;
1834         /* 27 38 28 */ coords[23 * dim + 0] = -dis;
1835         coords[23 * dim + 1]                = dis;
1836         coords[23 * dim + 2]                = 0.5;
1837       }
1838     }
1839     PetscCall(VecRestoreArray(coordinates, &coords));
1840     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1841     PetscCall(VecDestroy(&coordinates));
1842   }
1843   /* Create periodicity */
1844   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1845     PetscReal L[3]       = {-1., -1., 0.};
1846     PetscReal maxCell[3] = {-1., -1., 0.};
1847     PetscReal lower[3]   = {0.0, 0.0, 0.0};
1848     PetscReal upper[3]   = {1.0, 1.0, 1.5};
1849     PetscInt  numZCells  = 3;
1850 
1851     L[2]       = upper[2] - lower[2];
1852     maxCell[2] = 1.1 * (L[2] / numZCells);
1853     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1854   }
1855   {
1856     DM          cdm;
1857     PetscDS     cds;
1858     PetscScalar c[2] = {1.0, 1.0};
1859 
1860     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToCylinder));
1861     PetscCall(DMGetCoordinateDM(dm, &cdm));
1862     PetscCall(DMGetDS(cdm, &cds));
1863     PetscCall(PetscDSSetConstants(cds, 2, c));
1864   }
1865   /* Wait for coordinate creation before doing in-place modification */
1866   PetscCall(DMPlexInterpolateInPlace_Internal(dm));
1867   PetscFunctionReturn(0);
1868 }
1869 
1870 /*@
1871   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
1872 
1873   Collective
1874 
1875   Input Parameters:
1876 + comm      - The communicator for the `DM` object
1877 - periodicZ - The boundary type for the Z direction
1878 
1879   Output Parameter:
1880 . dm  - The DM object
1881 
1882   Level: beginner
1883 
1884   Note:
1885   Here is the output numbering looking from the bottom of the cylinder:
1886 .vb
1887        17-----14
1888         |     |
1889         |  2  |
1890         |     |
1891  17-----8-----7-----14
1892   |     |     |     |
1893   |  3  |  0  |  1  |
1894   |     |     |     |
1895  19-----5-----6-----13
1896         |     |
1897         |  4  |
1898         |     |
1899        19-----13
1900 
1901  and up through the top
1902 
1903        18-----16
1904         |     |
1905         |  2  |
1906         |     |
1907  18----10----11-----16
1908   |     |     |     |
1909   |  3  |  0  |  1  |
1910   |     |     |     |
1911  20-----9----12-----15
1912         |     |
1913         |  4  |
1914         |     |
1915        20-----15
1916 .ve
1917 
1918 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1919 @*/
1920 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, DMBoundaryType periodicZ, DM *dm)
1921 {
1922   PetscFunctionBegin;
1923   PetscValidPointer(dm, 3);
1924   PetscCall(DMCreate(comm, dm));
1925   PetscCall(DMSetType(*dm, DMPLEX));
1926   PetscCall(DMPlexCreateHexCylinderMesh_Internal(*dm, periodicZ));
1927   PetscFunctionReturn(0);
1928 }
1929 
1930 static PetscErrorCode DMPlexCreateWedgeCylinderMesh_Internal(DM dm, PetscInt n, PetscBool interpolate)
1931 {
1932   const PetscInt dim = 3;
1933   PetscInt       numCells, numVertices, v;
1934   PetscMPIInt    rank;
1935 
1936   PetscFunctionBegin;
1937   PetscCheck(n >= 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %" PetscInt_FMT " cannot be negative", n);
1938   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1939   PetscCall(DMSetDimension(dm, dim));
1940   /* Must create the celltype label here so that we do not automatically try to compute the types */
1941   PetscCall(DMCreateLabel(dm, "celltype"));
1942   /* Create topology */
1943   {
1944     PetscInt cone[6], c;
1945 
1946     numCells    = rank == 0 ? n : 0;
1947     numVertices = rank == 0 ? 2 * (n + 1) : 0;
1948     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1949     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
1950     PetscCall(DMSetUp(dm));
1951     for (c = 0; c < numCells; c++) {
1952       cone[0] = c + n * 1;
1953       cone[1] = (c + 1) % n + n * 1;
1954       cone[2] = 0 + 3 * n;
1955       cone[3] = c + n * 2;
1956       cone[4] = (c + 1) % n + n * 2;
1957       cone[5] = 1 + 3 * n;
1958       PetscCall(DMPlexSetCone(dm, c, cone));
1959       PetscCall(DMPlexSetCellType(dm, c, DM_POLYTOPE_TRI_PRISM_TENSOR));
1960     }
1961     PetscCall(DMPlexSymmetrize(dm));
1962     PetscCall(DMPlexStratify(dm));
1963   }
1964   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetCellType(dm, v, DM_POLYTOPE_POINT));
1965   /* Create cylinder geometry */
1966   {
1967     Vec          coordinates;
1968     PetscSection coordSection;
1969     PetscScalar *coords;
1970     PetscInt     coordSize, c;
1971 
1972     /* Build coordinates */
1973     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1974     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1975     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1976     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1977     for (v = numCells; v < numCells + numVertices; ++v) {
1978       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1979       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1980     }
1981     PetscCall(PetscSectionSetUp(coordSection));
1982     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1983     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1984     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1985     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1986     PetscCall(VecSetBlockSize(coordinates, dim));
1987     PetscCall(VecSetType(coordinates, VECSTANDARD));
1988     PetscCall(VecGetArray(coordinates, &coords));
1989     for (c = 0; c < numCells; c++) {
1990       coords[(c + 0 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
1991       coords[(c + 0 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
1992       coords[(c + 0 * n) * dim + 2] = 1.0;
1993       coords[(c + 1 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
1994       coords[(c + 1 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
1995       coords[(c + 1 * n) * dim + 2] = 0.0;
1996     }
1997     if (rank == 0) {
1998       coords[(2 * n + 0) * dim + 0] = 0.0;
1999       coords[(2 * n + 0) * dim + 1] = 0.0;
2000       coords[(2 * n + 0) * dim + 2] = 1.0;
2001       coords[(2 * n + 1) * dim + 0] = 0.0;
2002       coords[(2 * n + 1) * dim + 1] = 0.0;
2003       coords[(2 * n + 1) * dim + 2] = 0.0;
2004     }
2005     PetscCall(VecRestoreArray(coordinates, &coords));
2006     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2007     PetscCall(VecDestroy(&coordinates));
2008   }
2009   /* Interpolate */
2010   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2011   PetscFunctionReturn(0);
2012 }
2013 
2014 /*@
2015   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
2016 
2017   Collective
2018 
2019   Input Parameters:
2020 + comm - The communicator for the `DM` object
2021 . n    - The number of wedges around the origin
2022 - interpolate - Create edges and faces
2023 
2024   Output Parameter:
2025 . dm  - The `DM` object
2026 
2027   Level: beginner
2028 
2029 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2030 @*/
2031 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
2032 {
2033   PetscFunctionBegin;
2034   PetscValidPointer(dm, 4);
2035   PetscCall(DMCreate(comm, dm));
2036   PetscCall(DMSetType(*dm, DMPLEX));
2037   PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(*dm, n, interpolate));
2038   PetscFunctionReturn(0);
2039 }
2040 
2041 static inline PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2042 {
2043   PetscReal prod = 0.0;
2044   PetscInt  i;
2045   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
2046   return PetscSqrtReal(prod);
2047 }
2048 static inline PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2049 {
2050   PetscReal prod = 0.0;
2051   PetscInt  i;
2052   for (i = 0; i < dim; ++i) prod += x[i] * y[i];
2053   return prod;
2054 }
2055 
2056 /* The first constant is the sphere radius */
2057 static void snapToSphere(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
2058 {
2059   PetscReal r     = PetscRealPart(constants[0]);
2060   PetscReal norm2 = 0.0, fac;
2061   PetscInt  n     = uOff[1] - uOff[0], d;
2062 
2063   for (d = 0; d < n; ++d) norm2 += PetscSqr(PetscRealPart(u[d]));
2064   fac = r / PetscSqrtReal(norm2);
2065   for (d = 0; d < n; ++d) f0[d] = u[d] * fac;
2066 }
2067 
2068 static PetscErrorCode DMPlexCreateSphereMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, PetscReal R)
2069 {
2070   const PetscInt embedDim = dim + 1;
2071   PetscSection   coordSection;
2072   Vec            coordinates;
2073   PetscScalar   *coords;
2074   PetscReal     *coordsIn;
2075   PetscInt       numCells, numEdges, numVerts = 0, firstVertex = 0, v, firstEdge, coordSize, d, c, e;
2076   PetscMPIInt    rank;
2077 
2078   PetscFunctionBegin;
2079   PetscValidLogicalCollectiveBool(dm, simplex, 3);
2080   PetscCall(DMSetDimension(dm, dim));
2081   PetscCall(DMSetCoordinateDim(dm, dim + 1));
2082   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2083   switch (dim) {
2084   case 2:
2085     if (simplex) {
2086       const PetscReal radius    = PetscSqrtReal(1 + PETSC_PHI * PETSC_PHI) / (1.0 + PETSC_PHI);
2087       const PetscReal edgeLen   = 2.0 / (1.0 + PETSC_PHI) * (R / radius);
2088       const PetscInt  degree    = 5;
2089       PetscReal       vertex[3] = {0.0, 1.0 / (1.0 + PETSC_PHI), PETSC_PHI / (1.0 + PETSC_PHI)};
2090       PetscInt        s[3]      = {1, 1, 1};
2091       PetscInt        cone[3];
2092       PetscInt       *graph, p, i, j, k;
2093 
2094       vertex[0] *= R / radius;
2095       vertex[1] *= R / radius;
2096       vertex[2] *= R / radius;
2097       numCells    = rank == 0 ? 20 : 0;
2098       numVerts    = rank == 0 ? 12 : 0;
2099       firstVertex = numCells;
2100       /* Use icosahedron, which for a R-sphere has coordinates which are all cyclic permutations of
2101 
2102            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
2103 
2104          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2105          length is then given by 2/(1+\phi) = 2 * 0.38197 = 0.76393.
2106       */
2107       /* Construct vertices */
2108       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2109       if (rank == 0) {
2110         for (p = 0, i = 0; p < embedDim; ++p) {
2111           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2112             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2113               for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertex[(d + p) % embedDim];
2114               ++i;
2115             }
2116           }
2117         }
2118       }
2119       /* Construct graph */
2120       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2121       for (i = 0; i < numVerts; ++i) {
2122         for (j = 0, k = 0; j < numVerts; ++j) {
2123           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2124             graph[i * numVerts + j] = 1;
2125             ++k;
2126           }
2127         }
2128         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid icosahedron, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2129       }
2130       /* Build Topology */
2131       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2132       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2133       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2134       /* Cells */
2135       for (i = 0, c = 0; i < numVerts; ++i) {
2136         for (j = 0; j < i; ++j) {
2137           for (k = 0; k < j; ++k) {
2138             if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i]) {
2139               cone[0] = firstVertex + i;
2140               cone[1] = firstVertex + j;
2141               cone[2] = firstVertex + k;
2142               /* Check orientation */
2143               {
2144                 const PetscInt epsilon[3][3][3] = {
2145                   {{0, 0, 0},  {0, 0, 1},  {0, -1, 0}},
2146                   {{0, 0, -1}, {0, 0, 0},  {1, 0, 0} },
2147                   {{0, 1, 0},  {-1, 0, 0}, {0, 0, 0} }
2148                 };
2149                 PetscReal normal[3];
2150                 PetscInt  e, f;
2151 
2152                 for (d = 0; d < embedDim; ++d) {
2153                   normal[d] = 0.0;
2154                   for (e = 0; e < embedDim; ++e) {
2155                     for (f = 0; f < embedDim; ++f) normal[d] += epsilon[d][e][f] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]);
2156                   }
2157                 }
2158                 if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2159                   PetscInt tmp = cone[1];
2160                   cone[1]      = cone[2];
2161                   cone[2]      = tmp;
2162                 }
2163               }
2164               PetscCall(DMPlexSetCone(dm, c++, cone));
2165             }
2166           }
2167         }
2168       }
2169       PetscCall(DMPlexSymmetrize(dm));
2170       PetscCall(DMPlexStratify(dm));
2171       PetscCall(PetscFree(graph));
2172     } else {
2173       /*
2174         12-21--13
2175          |     |
2176         25  4  24
2177          |     |
2178   12-25--9-16--8-24--13
2179    |     |     |     |
2180   23  5 17  0 15  3  22
2181    |     |     |     |
2182   10-20--6-14--7-19--11
2183          |     |
2184         20  1  19
2185          |     |
2186         10-18--11
2187          |     |
2188         23  2  22
2189          |     |
2190         12-21--13
2191        */
2192       PetscInt cone[4], ornt[4];
2193 
2194       numCells    = rank == 0 ? 6 : 0;
2195       numEdges    = rank == 0 ? 12 : 0;
2196       numVerts    = rank == 0 ? 8 : 0;
2197       firstVertex = numCells;
2198       firstEdge   = numCells + numVerts;
2199       /* Build Topology */
2200       PetscCall(DMPlexSetChart(dm, 0, numCells + numEdges + numVerts));
2201       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 4));
2202       for (e = firstEdge; e < firstEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
2203       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2204       if (rank == 0) {
2205         /* Cell 0 */
2206         cone[0] = 14;
2207         cone[1] = 15;
2208         cone[2] = 16;
2209         cone[3] = 17;
2210         PetscCall(DMPlexSetCone(dm, 0, cone));
2211         ornt[0] = 0;
2212         ornt[1] = 0;
2213         ornt[2] = 0;
2214         ornt[3] = 0;
2215         PetscCall(DMPlexSetConeOrientation(dm, 0, ornt));
2216         /* Cell 1 */
2217         cone[0] = 18;
2218         cone[1] = 19;
2219         cone[2] = 14;
2220         cone[3] = 20;
2221         PetscCall(DMPlexSetCone(dm, 1, cone));
2222         ornt[0] = 0;
2223         ornt[1] = 0;
2224         ornt[2] = -1;
2225         ornt[3] = 0;
2226         PetscCall(DMPlexSetConeOrientation(dm, 1, ornt));
2227         /* Cell 2 */
2228         cone[0] = 21;
2229         cone[1] = 22;
2230         cone[2] = 18;
2231         cone[3] = 23;
2232         PetscCall(DMPlexSetCone(dm, 2, cone));
2233         ornt[0] = 0;
2234         ornt[1] = 0;
2235         ornt[2] = -1;
2236         ornt[3] = 0;
2237         PetscCall(DMPlexSetConeOrientation(dm, 2, ornt));
2238         /* Cell 3 */
2239         cone[0] = 19;
2240         cone[1] = 22;
2241         cone[2] = 24;
2242         cone[3] = 15;
2243         PetscCall(DMPlexSetCone(dm, 3, cone));
2244         ornt[0] = -1;
2245         ornt[1] = -1;
2246         ornt[2] = 0;
2247         ornt[3] = -1;
2248         PetscCall(DMPlexSetConeOrientation(dm, 3, ornt));
2249         /* Cell 4 */
2250         cone[0] = 16;
2251         cone[1] = 24;
2252         cone[2] = 21;
2253         cone[3] = 25;
2254         PetscCall(DMPlexSetCone(dm, 4, cone));
2255         ornt[0] = -1;
2256         ornt[1] = -1;
2257         ornt[2] = -1;
2258         ornt[3] = 0;
2259         PetscCall(DMPlexSetConeOrientation(dm, 4, ornt));
2260         /* Cell 5 */
2261         cone[0] = 20;
2262         cone[1] = 17;
2263         cone[2] = 25;
2264         cone[3] = 23;
2265         PetscCall(DMPlexSetCone(dm, 5, cone));
2266         ornt[0] = -1;
2267         ornt[1] = -1;
2268         ornt[2] = -1;
2269         ornt[3] = -1;
2270         PetscCall(DMPlexSetConeOrientation(dm, 5, ornt));
2271         /* Edges */
2272         cone[0] = 6;
2273         cone[1] = 7;
2274         PetscCall(DMPlexSetCone(dm, 14, cone));
2275         cone[0] = 7;
2276         cone[1] = 8;
2277         PetscCall(DMPlexSetCone(dm, 15, cone));
2278         cone[0] = 8;
2279         cone[1] = 9;
2280         PetscCall(DMPlexSetCone(dm, 16, cone));
2281         cone[0] = 9;
2282         cone[1] = 6;
2283         PetscCall(DMPlexSetCone(dm, 17, cone));
2284         cone[0] = 10;
2285         cone[1] = 11;
2286         PetscCall(DMPlexSetCone(dm, 18, cone));
2287         cone[0] = 11;
2288         cone[1] = 7;
2289         PetscCall(DMPlexSetCone(dm, 19, cone));
2290         cone[0] = 6;
2291         cone[1] = 10;
2292         PetscCall(DMPlexSetCone(dm, 20, cone));
2293         cone[0] = 12;
2294         cone[1] = 13;
2295         PetscCall(DMPlexSetCone(dm, 21, cone));
2296         cone[0] = 13;
2297         cone[1] = 11;
2298         PetscCall(DMPlexSetCone(dm, 22, cone));
2299         cone[0] = 10;
2300         cone[1] = 12;
2301         PetscCall(DMPlexSetCone(dm, 23, cone));
2302         cone[0] = 13;
2303         cone[1] = 8;
2304         PetscCall(DMPlexSetCone(dm, 24, cone));
2305         cone[0] = 12;
2306         cone[1] = 9;
2307         PetscCall(DMPlexSetCone(dm, 25, cone));
2308       }
2309       PetscCall(DMPlexSymmetrize(dm));
2310       PetscCall(DMPlexStratify(dm));
2311       /* Build coordinates */
2312       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2313       if (rank == 0) {
2314         coordsIn[0 * embedDim + 0] = -R;
2315         coordsIn[0 * embedDim + 1] = R;
2316         coordsIn[0 * embedDim + 2] = -R;
2317         coordsIn[1 * embedDim + 0] = R;
2318         coordsIn[1 * embedDim + 1] = R;
2319         coordsIn[1 * embedDim + 2] = -R;
2320         coordsIn[2 * embedDim + 0] = R;
2321         coordsIn[2 * embedDim + 1] = -R;
2322         coordsIn[2 * embedDim + 2] = -R;
2323         coordsIn[3 * embedDim + 0] = -R;
2324         coordsIn[3 * embedDim + 1] = -R;
2325         coordsIn[3 * embedDim + 2] = -R;
2326         coordsIn[4 * embedDim + 0] = -R;
2327         coordsIn[4 * embedDim + 1] = R;
2328         coordsIn[4 * embedDim + 2] = R;
2329         coordsIn[5 * embedDim + 0] = R;
2330         coordsIn[5 * embedDim + 1] = R;
2331         coordsIn[5 * embedDim + 2] = R;
2332         coordsIn[6 * embedDim + 0] = -R;
2333         coordsIn[6 * embedDim + 1] = -R;
2334         coordsIn[6 * embedDim + 2] = R;
2335         coordsIn[7 * embedDim + 0] = R;
2336         coordsIn[7 * embedDim + 1] = -R;
2337         coordsIn[7 * embedDim + 2] = R;
2338       }
2339     }
2340     break;
2341   case 3:
2342     if (simplex) {
2343       const PetscReal edgeLen         = 1.0 / PETSC_PHI;
2344       PetscReal       vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
2345       PetscReal       vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
2346       PetscReal       vertexC[4]      = {0.5, 0.5 * PETSC_PHI, 0.5 / PETSC_PHI, 0.0};
2347       const PetscInt  degree          = 12;
2348       PetscInt        s[4]            = {1, 1, 1};
2349       PetscInt        evenPerm[12][4] = {
2350         {0, 1, 2, 3},
2351         {0, 2, 3, 1},
2352         {0, 3, 1, 2},
2353         {1, 0, 3, 2},
2354         {1, 2, 0, 3},
2355         {1, 3, 2, 0},
2356         {2, 0, 1, 3},
2357         {2, 1, 3, 0},
2358         {2, 3, 0, 1},
2359         {3, 0, 2, 1},
2360         {3, 1, 0, 2},
2361         {3, 2, 1, 0}
2362       };
2363       PetscInt  cone[4];
2364       PetscInt *graph, p, i, j, k, l;
2365 
2366       vertexA[0] *= R;
2367       vertexA[1] *= R;
2368       vertexA[2] *= R;
2369       vertexA[3] *= R;
2370       vertexB[0] *= R;
2371       vertexB[1] *= R;
2372       vertexB[2] *= R;
2373       vertexB[3] *= R;
2374       vertexC[0] *= R;
2375       vertexC[1] *= R;
2376       vertexC[2] *= R;
2377       vertexC[3] *= R;
2378       numCells    = rank == 0 ? 600 : 0;
2379       numVerts    = rank == 0 ? 120 : 0;
2380       firstVertex = numCells;
2381       /* Use the 600-cell, which for a unit sphere has coordinates which are
2382 
2383            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
2384                (\pm 1,    0,       0,      0)  all cyclic permutations   8
2385            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
2386 
2387          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2388          length is then given by 1/\phi = 0.61803.
2389 
2390          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
2391          http://mathworld.wolfram.com/600-Cell.html
2392       */
2393       /* Construct vertices */
2394       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2395       i = 0;
2396       if (rank == 0) {
2397         for (s[0] = -1; s[0] < 2; s[0] += 2) {
2398           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2399             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2400               for (s[3] = -1; s[3] < 2; s[3] += 2) {
2401                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[d] * vertexA[d];
2402                 ++i;
2403               }
2404             }
2405           }
2406         }
2407         for (p = 0; p < embedDim; ++p) {
2408           s[1] = s[2] = s[3] = 1;
2409           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2410             for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertexB[(d + p) % embedDim];
2411             ++i;
2412           }
2413         }
2414         for (p = 0; p < 12; ++p) {
2415           s[3] = 1;
2416           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2417             for (s[1] = -1; s[1] < 2; s[1] += 2) {
2418               for (s[2] = -1; s[2] < 2; s[2] += 2) {
2419                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[evenPerm[p][d]] * vertexC[evenPerm[p][d]];
2420                 ++i;
2421               }
2422             }
2423           }
2424         }
2425       }
2426       PetscCheck(i == numVerts, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertices %" PetscInt_FMT " != %" PetscInt_FMT, i, numVerts);
2427       /* Construct graph */
2428       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2429       for (i = 0; i < numVerts; ++i) {
2430         for (j = 0, k = 0; j < numVerts; ++j) {
2431           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2432             graph[i * numVerts + j] = 1;
2433             ++k;
2434           }
2435         }
2436         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2437       }
2438       /* Build Topology */
2439       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2440       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2441       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2442       /* Cells */
2443       if (rank == 0) {
2444         for (i = 0, c = 0; i < numVerts; ++i) {
2445           for (j = 0; j < i; ++j) {
2446             for (k = 0; k < j; ++k) {
2447               for (l = 0; l < k; ++l) {
2448                 if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i] && graph[l * numVerts + i] && graph[l * numVerts + j] && graph[l * numVerts + k]) {
2449                   cone[0] = firstVertex + i;
2450                   cone[1] = firstVertex + j;
2451                   cone[2] = firstVertex + k;
2452                   cone[3] = firstVertex + l;
2453                   /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
2454                   {
2455                     const PetscInt epsilon[4][4][4][4] = {
2456                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, -1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 0, 0}, {0, 1, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 1, 0}, {0, -1, 0, 0}, {0, 0, 0, 0}}},
2457 
2458                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}}, {{0, 0, -1, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}}},
2459 
2460                       {{{0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, 0, 0}, {0, -1, 0, 0}}, {{0, 0, 0, -1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 1, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}},
2461 
2462                       {{{0, 0, 0, 0}, {0, 0, -1, 0}, {0, 1, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 1, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}}, {{0, -1, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}} }
2463                     };
2464                     PetscReal normal[4];
2465                     PetscInt  e, f, g;
2466 
2467                     for (d = 0; d < embedDim; ++d) {
2468                       normal[d] = 0.0;
2469                       for (e = 0; e < embedDim; ++e) {
2470                         for (f = 0; f < embedDim; ++f) {
2471                           for (g = 0; g < embedDim; ++g) {
2472                             normal[d] += epsilon[d][e][f][g] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]) * (coordsIn[l * embedDim + f] - coordsIn[i * embedDim + f]);
2473                           }
2474                         }
2475                       }
2476                     }
2477                     if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2478                       PetscInt tmp = cone[1];
2479                       cone[1]      = cone[2];
2480                       cone[2]      = tmp;
2481                     }
2482                   }
2483                   PetscCall(DMPlexSetCone(dm, c++, cone));
2484                 }
2485               }
2486             }
2487           }
2488         }
2489       }
2490       PetscCall(DMPlexSymmetrize(dm));
2491       PetscCall(DMPlexStratify(dm));
2492       PetscCall(PetscFree(graph));
2493     }
2494     break;
2495   default:
2496     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unsupported dimension for sphere: %" PetscInt_FMT, dim);
2497   }
2498   /* Create coordinates */
2499   PetscCall(DMGetCoordinateSection(dm, &coordSection));
2500   PetscCall(PetscSectionSetNumFields(coordSection, 1));
2501   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, embedDim));
2502   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVerts));
2503   for (v = firstVertex; v < firstVertex + numVerts; ++v) {
2504     PetscCall(PetscSectionSetDof(coordSection, v, embedDim));
2505     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, embedDim));
2506   }
2507   PetscCall(PetscSectionSetUp(coordSection));
2508   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2509   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2510   PetscCall(VecSetBlockSize(coordinates, embedDim));
2511   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2512   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2513   PetscCall(VecSetType(coordinates, VECSTANDARD));
2514   PetscCall(VecGetArray(coordinates, &coords));
2515   for (v = 0; v < numVerts; ++v)
2516     for (d = 0; d < embedDim; ++d) coords[v * embedDim + d] = coordsIn[v * embedDim + d];
2517   PetscCall(VecRestoreArray(coordinates, &coords));
2518   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2519   PetscCall(VecDestroy(&coordinates));
2520   PetscCall(PetscFree(coordsIn));
2521   {
2522     DM          cdm;
2523     PetscDS     cds;
2524     PetscScalar c = R;
2525 
2526     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToSphere));
2527     PetscCall(DMGetCoordinateDM(dm, &cdm));
2528     PetscCall(DMGetDS(cdm, &cds));
2529     PetscCall(PetscDSSetConstants(cds, 1, &c));
2530   }
2531   /* Wait for coordinate creation before doing in-place modification */
2532   if (simplex) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2533   PetscFunctionReturn(0);
2534 }
2535 
2536 typedef void (*TPSEvaluateFunc)(const PetscReal[], PetscReal *, PetscReal[], PetscReal (*)[3]);
2537 
2538 /*
2539  The Schwarz P implicit surface is
2540 
2541      f(x) = cos(x0) + cos(x1) + cos(x2) = 0
2542 */
2543 static void TPSEvaluate_SchwarzP(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2544 {
2545   PetscReal c[3] = {PetscCosReal(y[0] * PETSC_PI), PetscCosReal(y[1] * PETSC_PI), PetscCosReal(y[2] * PETSC_PI)};
2546   PetscReal g[3] = {-PetscSinReal(y[0] * PETSC_PI), -PetscSinReal(y[1] * PETSC_PI), -PetscSinReal(y[2] * PETSC_PI)};
2547   f[0]           = c[0] + c[1] + c[2];
2548   for (PetscInt i = 0; i < 3; i++) {
2549     grad[i] = PETSC_PI * g[i];
2550     for (PetscInt j = 0; j < 3; j++) hess[i][j] = (i == j) ? -PetscSqr(PETSC_PI) * c[i] : 0.;
2551   }
2552 }
2553 
2554 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2555 static PetscErrorCode TPSExtrudeNormalFunc_SchwarzP(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2556 {
2557   for (PetscInt i = 0; i < 3; i++) u[i] = -PETSC_PI * PetscSinReal(x[i] * PETSC_PI);
2558   return 0;
2559 }
2560 
2561 /*
2562  The Gyroid implicit surface is
2563 
2564  f(x,y,z) = sin(pi * x) * cos (pi * (y + 1/2))  + sin(pi * (y + 1/2)) * cos(pi * (z + 1/4)) + sin(pi * (z + 1/4)) * cos(pi * x)
2565 
2566 */
2567 static void TPSEvaluate_Gyroid(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2568 {
2569   PetscReal s[3] = {PetscSinReal(PETSC_PI * y[0]), PetscSinReal(PETSC_PI * (y[1] + .5)), PetscSinReal(PETSC_PI * (y[2] + .25))};
2570   PetscReal c[3] = {PetscCosReal(PETSC_PI * y[0]), PetscCosReal(PETSC_PI * (y[1] + .5)), PetscCosReal(PETSC_PI * (y[2] + .25))};
2571   f[0]           = s[0] * c[1] + s[1] * c[2] + s[2] * c[0];
2572   grad[0]        = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2573   grad[1]        = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2574   grad[2]        = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2575   hess[0][0]     = -PetscSqr(PETSC_PI) * (s[0] * c[1] + s[2] * c[0]);
2576   hess[0][1]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2577   hess[0][2]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2578   hess[1][0]     = -PetscSqr(PETSC_PI) * (s[1] * c[2] + s[0] * c[1]);
2579   hess[1][1]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2580   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2581   hess[2][0]     = -PetscSqr(PETSC_PI) * (s[2] * c[0] + s[1] * c[2]);
2582   hess[2][1]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2583   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2584 }
2585 
2586 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2587 static PetscErrorCode TPSExtrudeNormalFunc_Gyroid(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2588 {
2589   PetscReal s[3] = {PetscSinReal(PETSC_PI * x[0]), PetscSinReal(PETSC_PI * (x[1] + .5)), PetscSinReal(PETSC_PI * (x[2] + .25))};
2590   PetscReal c[3] = {PetscCosReal(PETSC_PI * x[0]), PetscCosReal(PETSC_PI * (x[1] + .5)), PetscCosReal(PETSC_PI * (x[2] + .25))};
2591   u[0]           = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2592   u[1]           = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2593   u[2]           = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2594   return 0;
2595 }
2596 
2597 /*
2598    We wish to solve
2599 
2600          min_y || y - x ||^2  subject to f(y) = 0
2601 
2602    Let g(y) = grad(f).  The minimization problem is equivalent to asking to satisfy
2603    f(y) = 0 and (y-x) is parallel to g(y).  We do this by using Householder QR to obtain a basis for the
2604    tangent space and ask for both components in the tangent space to be zero.
2605 
2606    Take g to be a column vector and compute the "full QR" factorization Q R = g,
2607    where Q = I - 2 n n^T is a symmetric orthogonal matrix.
2608    The first column of Q is parallel to g so the remaining two columns span the null space.
2609    Let Qn = Q[:,1:] be those remaining columns.  Then Qn Qn^T is an orthogonal projector into the tangent space.
2610    Since Q is symmetric, this is equivalent to multipyling by Q and taking the last two entries.
2611    In total, we have a system of 3 equations in 3 unknowns:
2612 
2613      f(y) = 0                       1 equation
2614      Qn^T (y - x) = 0               2 equations
2615 
2616    Here, we compute the residual and Jacobian of this system.
2617 */
2618 static void TPSNearestPointResJac(TPSEvaluateFunc feval, const PetscScalar x[], const PetscScalar y[], PetscScalar res[], PetscScalar J[])
2619 {
2620   PetscReal yreal[3] = {PetscRealPart(y[0]), PetscRealPart(y[1]), PetscRealPart(y[2])};
2621   PetscReal d[3]     = {PetscRealPart(y[0] - x[0]), PetscRealPart(y[1] - x[1]), PetscRealPart(y[2] - x[2])};
2622   PetscReal f, grad[3], n[3], norm, norm_y[3], nd, nd_y[3], sign;
2623   PetscReal n_y[3][3] = {
2624     {0, 0, 0},
2625     {0, 0, 0},
2626     {0, 0, 0}
2627   };
2628 
2629   feval(yreal, &f, grad, n_y);
2630 
2631   for (PetscInt i = 0; i < 3; i++) n[i] = grad[i];
2632   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2633   for (PetscInt i = 0; i < 3; i++) norm_y[i] = 1. / norm * n[i] * n_y[i][i];
2634 
2635   // Define the Householder reflector
2636   sign = n[0] >= 0 ? 1. : -1.;
2637   n[0] += norm * sign;
2638   for (PetscInt i = 0; i < 3; i++) n_y[0][i] += norm_y[i] * sign;
2639 
2640   norm      = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2641   norm_y[0] = 1. / norm * (n[0] * n_y[0][0]);
2642   norm_y[1] = 1. / norm * (n[0] * n_y[0][1] + n[1] * n_y[1][1]);
2643   norm_y[2] = 1. / norm * (n[0] * n_y[0][2] + n[2] * n_y[2][2]);
2644 
2645   for (PetscInt i = 0; i < 3; i++) {
2646     n[i] /= norm;
2647     for (PetscInt j = 0; j < 3; j++) {
2648       // note that n[i] is n_old[i]/norm when executing the code below
2649       n_y[i][j] = n_y[i][j] / norm - n[i] / norm * norm_y[j];
2650     }
2651   }
2652 
2653   nd = n[0] * d[0] + n[1] * d[1] + n[2] * d[2];
2654   for (PetscInt i = 0; i < 3; i++) nd_y[i] = n[i] + n_y[0][i] * d[0] + n_y[1][i] * d[1] + n_y[2][i] * d[2];
2655 
2656   res[0] = f;
2657   res[1] = d[1] - 2 * n[1] * nd;
2658   res[2] = d[2] - 2 * n[2] * nd;
2659   // J[j][i] is J_{ij} (column major)
2660   for (PetscInt j = 0; j < 3; j++) {
2661     J[0 + j * 3] = grad[j];
2662     J[1 + j * 3] = (j == 1) * 1. - 2 * (n_y[1][j] * nd + n[1] * nd_y[j]);
2663     J[2 + j * 3] = (j == 2) * 1. - 2 * (n_y[2][j] * nd + n[2] * nd_y[j]);
2664   }
2665 }
2666 
2667 /*
2668    Project x to the nearest point on the implicit surface using Newton's method.
2669 */
2670 static PetscErrorCode TPSNearestPoint(TPSEvaluateFunc feval, PetscScalar x[])
2671 {
2672   PetscScalar y[3] = {x[0], x[1], x[2]}; // Initial guess
2673 
2674   PetscFunctionBegin;
2675   for (PetscInt iter = 0; iter < 10; iter++) {
2676     PetscScalar res[3], J[9];
2677     PetscReal   resnorm;
2678     TPSNearestPointResJac(feval, x, y, res, J);
2679     resnorm = PetscSqrtReal(PetscSqr(PetscRealPart(res[0])) + PetscSqr(PetscRealPart(res[1])) + PetscSqr(PetscRealPart(res[2])));
2680     if (0) { // Turn on this monitor if you need to confirm quadratic convergence
2681       PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%" PetscInt_FMT "] res [%g %g %g]\n", iter, (double)PetscRealPart(res[0]), (double)PetscRealPart(res[1]), (double)PetscRealPart(res[2])));
2682     }
2683     if (resnorm < PETSC_SMALL) break;
2684 
2685     // Take the Newton step
2686     PetscCall(PetscKernel_A_gets_inverse_A_3(J, 0., PETSC_FALSE, NULL));
2687     PetscKernel_v_gets_v_minus_A_times_w_3(y, J, res);
2688   }
2689   for (PetscInt i = 0; i < 3; i++) x[i] = y[i];
2690   PetscFunctionReturn(0);
2691 }
2692 
2693 const char *const DMPlexTPSTypes[] = {"SCHWARZ_P", "GYROID", "DMPlexTPSType", "DMPLEX_TPS_", NULL};
2694 
2695 static PetscErrorCode DMPlexCreateTPSMesh_Internal(DM dm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness)
2696 {
2697   PetscMPIInt rank;
2698   PetscInt    topoDim = 2, spaceDim = 3, numFaces = 0, numVertices = 0, numEdges = 0;
2699   PetscInt(*edges)[2] = NULL, *edgeSets = NULL;
2700   PetscInt            *cells_flat = NULL;
2701   PetscReal           *vtxCoords  = NULL;
2702   TPSEvaluateFunc      evalFunc   = NULL;
2703   PetscSimplePointFunc normalFunc = NULL;
2704   DMLabel              label;
2705 
2706   PetscFunctionBegin;
2707   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2708   PetscCheck((layers != 0) ^ (thickness == 0.), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Layers %" PetscInt_FMT " must be nonzero iff thickness %g is nonzero", layers, (double)thickness);
2709   switch (tpstype) {
2710   case DMPLEX_TPS_SCHWARZ_P:
2711     PetscCheck(!periodic || (periodic[0] == DM_BOUNDARY_NONE && periodic[1] == DM_BOUNDARY_NONE && periodic[2] == DM_BOUNDARY_NONE), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Schwarz P does not support periodic meshes");
2712     if (rank == 0) {
2713       PetscInt(*cells)[6][4][4] = NULL; // [junction, junction-face, cell, conn]
2714       PetscInt  Njunctions = 0, Ncuts = 0, Npipes[3], vcount;
2715       PetscReal L = 1;
2716 
2717       Npipes[0]   = (extent[0] + 1) * extent[1] * extent[2];
2718       Npipes[1]   = extent[0] * (extent[1] + 1) * extent[2];
2719       Npipes[2]   = extent[0] * extent[1] * (extent[2] + 1);
2720       Njunctions  = extent[0] * extent[1] * extent[2];
2721       Ncuts       = 2 * (extent[0] * extent[1] + extent[1] * extent[2] + extent[2] * extent[0]);
2722       numVertices = 4 * (Npipes[0] + Npipes[1] + Npipes[2]) + 8 * Njunctions;
2723       PetscCall(PetscMalloc1(3 * numVertices, &vtxCoords));
2724       PetscCall(PetscMalloc1(Njunctions, &cells));
2725       PetscCall(PetscMalloc1(Ncuts * 4, &edges));
2726       PetscCall(PetscMalloc1(Ncuts * 4, &edgeSets));
2727       // x-normal pipes
2728       vcount = 0;
2729       for (PetscInt i = 0; i < extent[0] + 1; i++) {
2730         for (PetscInt j = 0; j < extent[1]; j++) {
2731           for (PetscInt k = 0; k < extent[2]; k++) {
2732             for (PetscInt l = 0; l < 4; l++) {
2733               vtxCoords[vcount++] = (2 * i - 1) * L;
2734               vtxCoords[vcount++] = 2 * j * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2735               vtxCoords[vcount++] = 2 * k * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2736             }
2737           }
2738         }
2739       }
2740       // y-normal pipes
2741       for (PetscInt i = 0; i < extent[0]; i++) {
2742         for (PetscInt j = 0; j < extent[1] + 1; j++) {
2743           for (PetscInt k = 0; k < extent[2]; k++) {
2744             for (PetscInt l = 0; l < 4; l++) {
2745               vtxCoords[vcount++] = 2 * i * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2746               vtxCoords[vcount++] = (2 * j - 1) * L;
2747               vtxCoords[vcount++] = 2 * k * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2748             }
2749           }
2750         }
2751       }
2752       // z-normal pipes
2753       for (PetscInt i = 0; i < extent[0]; i++) {
2754         for (PetscInt j = 0; j < extent[1]; j++) {
2755           for (PetscInt k = 0; k < extent[2] + 1; k++) {
2756             for (PetscInt l = 0; l < 4; l++) {
2757               vtxCoords[vcount++] = 2 * i * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2758               vtxCoords[vcount++] = 2 * j * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2759               vtxCoords[vcount++] = (2 * k - 1) * L;
2760             }
2761           }
2762         }
2763       }
2764       // junctions
2765       for (PetscInt i = 0; i < extent[0]; i++) {
2766         for (PetscInt j = 0; j < extent[1]; j++) {
2767           for (PetscInt k = 0; k < extent[2]; k++) {
2768             const PetscInt J = (i * extent[1] + j) * extent[2] + k, Jvoff = (Npipes[0] + Npipes[1] + Npipes[2]) * 4 + J * 8;
2769             PetscCheck(vcount / 3 == Jvoff, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected vertex count");
2770             for (PetscInt ii = 0; ii < 2; ii++) {
2771               for (PetscInt jj = 0; jj < 2; jj++) {
2772                 for (PetscInt kk = 0; kk < 2; kk++) {
2773                   double Ls           = (1 - sqrt(2) / 4) * L;
2774                   vtxCoords[vcount++] = 2 * i * L + (2 * ii - 1) * Ls;
2775                   vtxCoords[vcount++] = 2 * j * L + (2 * jj - 1) * Ls;
2776                   vtxCoords[vcount++] = 2 * k * L + (2 * kk - 1) * Ls;
2777                 }
2778               }
2779             }
2780             const PetscInt jfaces[3][2][4] = {
2781               {{3, 1, 0, 2}, {7, 5, 4, 6}}, // x-aligned
2782               {{5, 4, 0, 1}, {7, 6, 2, 3}}, // y-aligned
2783               {{6, 2, 0, 4}, {7, 3, 1, 5}}  // z-aligned
2784             };
2785             const PetscInt pipe_lo[3] = {// vertex numbers of pipes
2786                                          ((i * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + Npipes[0] + Npipes[1]) * 4};
2787             const PetscInt pipe_hi[3] = {// vertex numbers of pipes
2788                                          (((i + 1) * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j + 1) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + 1 + Npipes[0] + Npipes[1]) * 4};
2789             for (PetscInt dir = 0; dir < 3; dir++) { // x,y,z
2790               const PetscInt ijk[3] = {i, j, k};
2791               for (PetscInt l = 0; l < 4; l++) { // rotations
2792                 cells[J][dir * 2 + 0][l][0] = pipe_lo[dir] + l;
2793                 cells[J][dir * 2 + 0][l][1] = Jvoff + jfaces[dir][0][l];
2794                 cells[J][dir * 2 + 0][l][2] = Jvoff + jfaces[dir][0][(l - 1 + 4) % 4];
2795                 cells[J][dir * 2 + 0][l][3] = pipe_lo[dir] + (l - 1 + 4) % 4;
2796                 cells[J][dir * 2 + 1][l][0] = Jvoff + jfaces[dir][1][l];
2797                 cells[J][dir * 2 + 1][l][1] = pipe_hi[dir] + l;
2798                 cells[J][dir * 2 + 1][l][2] = pipe_hi[dir] + (l - 1 + 4) % 4;
2799                 cells[J][dir * 2 + 1][l][3] = Jvoff + jfaces[dir][1][(l - 1 + 4) % 4];
2800                 if (ijk[dir] == 0) {
2801                   edges[numEdges][0] = pipe_lo[dir] + l;
2802                   edges[numEdges][1] = pipe_lo[dir] + (l + 1) % 4;
2803                   edgeSets[numEdges] = dir * 2 + 1;
2804                   numEdges++;
2805                 }
2806                 if (ijk[dir] + 1 == extent[dir]) {
2807                   edges[numEdges][0] = pipe_hi[dir] + l;
2808                   edges[numEdges][1] = pipe_hi[dir] + (l + 1) % 4;
2809                   edgeSets[numEdges] = dir * 2 + 2;
2810                   numEdges++;
2811                 }
2812               }
2813             }
2814           }
2815         }
2816       }
2817       PetscCheck(numEdges == Ncuts * 4, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Edge count %" PetscInt_FMT " incompatible with number of cuts %" PetscInt_FMT, numEdges, Ncuts);
2818       numFaces   = 24 * Njunctions;
2819       cells_flat = cells[0][0][0];
2820     }
2821     evalFunc   = TPSEvaluate_SchwarzP;
2822     normalFunc = TPSExtrudeNormalFunc_SchwarzP;
2823     break;
2824   case DMPLEX_TPS_GYROID:
2825     if (rank == 0) {
2826       // This is a coarse mesh approximation of the gyroid shifted to being the zero of the level set
2827       //
2828       //     sin(pi*x)*cos(pi*(y+1/2)) + sin(pi*(y+1/2))*cos(pi*(z+1/4)) + sin(pi*(z+1/4))*cos(x)
2829       //
2830       // on the cell [0,2]^3.
2831       //
2832       // Think about dividing that cell into four columns, and focus on the column [0,1]x[0,1]x[0,2].
2833       // If you looked at the gyroid in that column at different slices of z you would see that it kind of spins
2834       // like a boomerang:
2835       //
2836       //     z = 0          z = 1/4        z = 1/2        z = 3/4     //
2837       //     -----          -------        -------        -------     //
2838       //                                                              //
2839       //     +       +      +       +      +       +      +   \   +   //
2840       //      \                                   /            \      //
2841       //       \            `-_   _-'            /              }     //
2842       //        *-_            `-'            _-'              /      //
2843       //     +     `-+      +       +      +-'     +      +   /   +   //
2844       //                                                              //
2845       //                                                              //
2846       //     z = 1          z = 5/4        z = 3/2        z = 7/4     //
2847       //     -----          -------        -------        -------     //
2848       //                                                              //
2849       //     +-_     +      +       +      +     _-+      +   /   +   //
2850       //        `-_            _-_            _-`            /        //
2851       //           \        _-'   `-_        /              {         //
2852       //            \                       /                \        //
2853       //     +       +      +       +      +       +      +   \   +   //
2854       //
2855       //
2856       // This course mesh approximates each of these slices by two line segments,
2857       // and then connects the segments in consecutive layers with quadrilateral faces.
2858       // All of the end points of the segments are multiples of 1/4 except for the
2859       // point * in the picture for z = 0 above and the similar points in other layers.
2860       // That point is at (gamma, gamma, 0), where gamma is calculated below.
2861       //
2862       // The column  [1,2]x[1,2]x[0,2] looks the same as this column;
2863       // The columns [1,2]x[0,1]x[0,2] and [0,1]x[1,2]x[0,2] are mirror images.
2864       //
2865       // As for how this method turned into the names given to the vertices:
2866       // that was not systematic, it was just the way it worked out in my handwritten notes.
2867 
2868       PetscInt facesPerBlock = 64;
2869       PetscInt vertsPerBlock = 56;
2870       PetscInt extentPlus[3];
2871       PetscInt numBlocks, numBlocksPlus;
2872       const PetscInt A = 0, B = 1, C = 2, D = 3, E = 4, F = 5, G = 6, H = 7, II = 8, J = 9, K = 10, L = 11, M = 12, N = 13, O = 14, P = 15, Q = 16, R = 17, S = 18, T = 19, U = 20, V = 21, W = 22, X = 23, Y = 24, Z = 25, Ap = 26, Bp = 27, Cp = 28, Dp = 29, Ep = 30, Fp = 31, Gp = 32, Hp = 33, Ip = 34, Jp = 35, Kp = 36, Lp = 37, Mp = 38, Np = 39, Op = 40, Pp = 41, Qp = 42, Rp = 43, Sp = 44, Tp = 45, Up = 46, Vp = 47, Wp = 48, Xp = 49, Yp = 50, Zp = 51, Aq = 52, Bq = 53, Cq = 54, Dq = 55;
2873       const PetscInt pattern[64][4] = {
2874   /* face to vertex within the coarse discretization of a single gyroid block */
2875   /* layer 0 */
2876         {A,           C,           K,           G          },
2877         {C,           B,           II,          K          },
2878         {D,           A,           H,           L          },
2879         {B + 56 * 1,  D,           L,           J          },
2880         {E,           B + 56 * 1,  J,           N          },
2881         {A + 56 * 2,  E,           N,           H + 56 * 2 },
2882         {F,           A + 56 * 2,  G + 56 * 2,  M          },
2883         {B,           F,           M,           II         },
2884  /* layer 1 */
2885         {G,           K,           Q,           O          },
2886         {K,           II,          P,           Q          },
2887         {L,           H,           O + 56 * 1,  R          },
2888         {J,           L,           R,           P          },
2889         {N,           J,           P,           S          },
2890         {H + 56 * 2,  N,           S,           O + 56 * 3 },
2891         {M,           G + 56 * 2,  O + 56 * 2,  T          },
2892         {II,          M,           T,           P          },
2893  /* layer 2 */
2894         {O,           Q,           Y,           U          },
2895         {Q,           P,           W,           Y          },
2896         {R,           O + 56 * 1,  U + 56 * 1,  Ap         },
2897         {P,           R,           Ap,          W          },
2898         {S,           P,           X,           Bp         },
2899         {O + 56 * 3,  S,           Bp,          V + 56 * 1 },
2900         {T,           O + 56 * 2,  V,           Z          },
2901         {P,           T,           Z,           X          },
2902  /* layer 3 */
2903         {U,           Y,           Ep,          Dp         },
2904         {Y,           W,           Cp,          Ep         },
2905         {Ap,          U + 56 * 1,  Dp + 56 * 1, Gp         },
2906         {W,           Ap,          Gp,          Cp         },
2907         {Bp,          X,           Cp + 56 * 2, Fp         },
2908         {V + 56 * 1,  Bp,          Fp,          Dp + 56 * 1},
2909         {Z,           V,           Dp,          Hp         },
2910         {X,           Z,           Hp,          Cp + 56 * 2},
2911  /* layer 4 */
2912         {Dp,          Ep,          Mp,          Kp         },
2913         {Ep,          Cp,          Ip,          Mp         },
2914         {Gp,          Dp + 56 * 1, Lp,          Np         },
2915         {Cp,          Gp,          Np,          Jp         },
2916         {Fp,          Cp + 56 * 2, Jp + 56 * 2, Pp         },
2917         {Dp + 56 * 1, Fp,          Pp,          Lp         },
2918         {Hp,          Dp,          Kp,          Op         },
2919         {Cp + 56 * 2, Hp,          Op,          Ip + 56 * 2},
2920  /* layer 5 */
2921         {Kp,          Mp,          Sp,          Rp         },
2922         {Mp,          Ip,          Qp,          Sp         },
2923         {Np,          Lp,          Rp,          Tp         },
2924         {Jp,          Np,          Tp,          Qp + 56 * 1},
2925         {Pp,          Jp + 56 * 2, Qp + 56 * 3, Up         },
2926         {Lp,          Pp,          Up,          Rp         },
2927         {Op,          Kp,          Rp,          Vp         },
2928         {Ip + 56 * 2, Op,          Vp,          Qp + 56 * 2},
2929  /* layer 6 */
2930         {Rp,          Sp,          Aq,          Yp         },
2931         {Sp,          Qp,          Wp,          Aq         },
2932         {Tp,          Rp,          Yp,          Cq         },
2933         {Qp + 56 * 1, Tp,          Cq,          Wp + 56 * 1},
2934         {Up,          Qp + 56 * 3, Xp + 56 * 1, Dq         },
2935         {Rp,          Up,          Dq,          Zp         },
2936         {Vp,          Rp,          Zp,          Bq         },
2937         {Qp + 56 * 2, Vp,          Bq,          Xp         },
2938  /* layer 7 (the top is the periodic image of the bottom of layer 0) */
2939         {Yp,          Aq,          C + 56 * 4,  A + 56 * 4 },
2940         {Aq,          Wp,          B + 56 * 4,  C + 56 * 4 },
2941         {Cq,          Yp,          A + 56 * 4,  D + 56 * 4 },
2942         {Wp + 56 * 1, Cq,          D + 56 * 4,  B + 56 * 5 },
2943         {Dq,          Xp + 56 * 1, B + 56 * 5,  E + 56 * 4 },
2944         {Zp,          Dq,          E + 56 * 4,  A + 56 * 6 },
2945         {Bq,          Zp,          A + 56 * 6,  F + 56 * 4 },
2946         {Xp,          Bq,          F + 56 * 4,  B + 56 * 4 }
2947       };
2948       const PetscReal gamma                = PetscAcosReal((PetscSqrtReal(3.) - 1.) / PetscSqrtReal(2.)) / PETSC_PI;
2949       const PetscReal patternCoords[56][3] = {
2950         {1.,        0.,        0.  }, /* A  */
2951         {0.,        1.,        0.  }, /* B  */
2952         {gamma,     gamma,     0.  }, /* C  */
2953         {1 + gamma, 1 - gamma, 0.  }, /* D  */
2954         {2 - gamma, 2 - gamma, 0.  }, /* E  */
2955         {1 - gamma, 1 + gamma, 0.  }, /* F  */
2956 
2957         {.5,        0,         .25 }, /* G  */
2958         {1.5,       0.,        .25 }, /* H  */
2959         {.5,        1.,        .25 }, /* II */
2960         {1.5,       1.,        .25 }, /* J  */
2961         {.25,       .5,        .25 }, /* K  */
2962         {1.25,      .5,        .25 }, /* L  */
2963         {.75,       1.5,       .25 }, /* M  */
2964         {1.75,      1.5,       .25 }, /* N  */
2965 
2966         {0.,        0.,        .5  }, /* O  */
2967         {1.,        1.,        .5  }, /* P  */
2968         {gamma,     1 - gamma, .5  }, /* Q  */
2969         {1 + gamma, gamma,     .5  }, /* R  */
2970         {2 - gamma, 1 + gamma, .5  }, /* S  */
2971         {1 - gamma, 2 - gamma, .5  }, /* T  */
2972 
2973         {0.,        .5,        .75 }, /* U  */
2974         {0.,        1.5,       .75 }, /* V  */
2975         {1.,        .5,        .75 }, /* W  */
2976         {1.,        1.5,       .75 }, /* X  */
2977         {.5,        .75,       .75 }, /* Y  */
2978         {.5,        1.75,      .75 }, /* Z  */
2979         {1.5,       .25,       .75 }, /* Ap */
2980         {1.5,       1.25,      .75 }, /* Bp */
2981 
2982         {1.,        0.,        1.  }, /* Cp */
2983         {0.,        1.,        1.  }, /* Dp */
2984         {1 - gamma, 1 - gamma, 1.  }, /* Ep */
2985         {1 + gamma, 1 + gamma, 1.  }, /* Fp */
2986         {2 - gamma, gamma,     1.  }, /* Gp */
2987         {gamma,     2 - gamma, 1.  }, /* Hp */
2988 
2989         {.5,        0.,        1.25}, /* Ip */
2990         {1.5,       0.,        1.25}, /* Jp */
2991         {.5,        1.,        1.25}, /* Kp */
2992         {1.5,       1.,        1.25}, /* Lp */
2993         {.75,       .5,        1.25}, /* Mp */
2994         {1.75,      .5,        1.25}, /* Np */
2995         {.25,       1.5,       1.25}, /* Op */
2996         {1.25,      1.5,       1.25}, /* Pp */
2997 
2998         {0.,        0.,        1.5 }, /* Qp */
2999         {1.,        1.,        1.5 }, /* Rp */
3000         {1 - gamma, gamma,     1.5 }, /* Sp */
3001         {2 - gamma, 1 - gamma, 1.5 }, /* Tp */
3002         {1 + gamma, 2 - gamma, 1.5 }, /* Up */
3003         {gamma,     1 + gamma, 1.5 }, /* Vp */
3004 
3005         {0.,        .5,        1.75}, /* Wp */
3006         {0.,        1.5,       1.75}, /* Xp */
3007         {1.,        .5,        1.75}, /* Yp */
3008         {1.,        1.5,       1.75}, /* Zp */
3009         {.5,        .25,       1.75}, /* Aq */
3010         {.5,        1.25,      1.75}, /* Bq */
3011         {1.5,       .75,       1.75}, /* Cq */
3012         {1.5,       1.75,      1.75}, /* Dq */
3013       };
3014       PetscInt(*cells)[64][4] = NULL;
3015       PetscBool *seen;
3016       PetscInt  *vertToTrueVert;
3017       PetscInt   count;
3018 
3019       for (PetscInt i = 0; i < 3; i++) extentPlus[i] = extent[i] + 1;
3020       numBlocks = 1;
3021       for (PetscInt i = 0; i < 3; i++) numBlocks *= extent[i];
3022       numBlocksPlus = 1;
3023       for (PetscInt i = 0; i < 3; i++) numBlocksPlus *= extentPlus[i];
3024       numFaces = numBlocks * facesPerBlock;
3025       PetscCall(PetscMalloc1(numBlocks, &cells));
3026       PetscCall(PetscCalloc1(numBlocksPlus * vertsPerBlock, &seen));
3027       for (PetscInt k = 0; k < extent[2]; k++) {
3028         for (PetscInt j = 0; j < extent[1]; j++) {
3029           for (PetscInt i = 0; i < extent[0]; i++) {
3030             for (PetscInt f = 0; f < facesPerBlock; f++) {
3031               for (PetscInt v = 0; v < 4; v++) {
3032                 PetscInt vertRaw     = pattern[f][v];
3033                 PetscInt blockidx    = vertRaw / 56;
3034                 PetscInt patternvert = vertRaw % 56;
3035                 PetscInt xplus       = (blockidx & 1);
3036                 PetscInt yplus       = (blockidx & 2) >> 1;
3037                 PetscInt zplus       = (blockidx & 4) >> 2;
3038                 PetscInt zcoord      = (periodic && periodic[2] == DM_BOUNDARY_PERIODIC) ? ((k + zplus) % extent[2]) : (k + zplus);
3039                 PetscInt ycoord      = (periodic && periodic[1] == DM_BOUNDARY_PERIODIC) ? ((j + yplus) % extent[1]) : (j + yplus);
3040                 PetscInt xcoord      = (periodic && periodic[0] == DM_BOUNDARY_PERIODIC) ? ((i + xplus) % extent[0]) : (i + xplus);
3041                 PetscInt vert        = ((zcoord * extentPlus[1] + ycoord) * extentPlus[0] + xcoord) * 56 + patternvert;
3042 
3043                 cells[(k * extent[1] + j) * extent[0] + i][f][v] = vert;
3044                 seen[vert]                                       = PETSC_TRUE;
3045               }
3046             }
3047           }
3048         }
3049       }
3050       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++)
3051         if (seen[i]) numVertices++;
3052       count = 0;
3053       PetscCall(PetscMalloc1(numBlocksPlus * vertsPerBlock, &vertToTrueVert));
3054       PetscCall(PetscMalloc1(numVertices * 3, &vtxCoords));
3055       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) vertToTrueVert[i] = -1;
3056       for (PetscInt k = 0; k < extentPlus[2]; k++) {
3057         for (PetscInt j = 0; j < extentPlus[1]; j++) {
3058           for (PetscInt i = 0; i < extentPlus[0]; i++) {
3059             for (PetscInt v = 0; v < vertsPerBlock; v++) {
3060               PetscInt vIdx = ((k * extentPlus[1] + j) * extentPlus[0] + i) * vertsPerBlock + v;
3061 
3062               if (seen[vIdx]) {
3063                 PetscInt thisVert;
3064 
3065                 vertToTrueVert[vIdx] = thisVert = count++;
3066 
3067                 for (PetscInt d = 0; d < 3; d++) vtxCoords[3 * thisVert + d] = patternCoords[v][d];
3068                 vtxCoords[3 * thisVert + 0] += i * 2;
3069                 vtxCoords[3 * thisVert + 1] += j * 2;
3070                 vtxCoords[3 * thisVert + 2] += k * 2;
3071               }
3072             }
3073           }
3074         }
3075       }
3076       for (PetscInt i = 0; i < numBlocks; i++) {
3077         for (PetscInt f = 0; f < facesPerBlock; f++) {
3078           for (PetscInt v = 0; v < 4; v++) cells[i][f][v] = vertToTrueVert[cells[i][f][v]];
3079         }
3080       }
3081       PetscCall(PetscFree(vertToTrueVert));
3082       PetscCall(PetscFree(seen));
3083       cells_flat = cells[0][0];
3084       numEdges   = 0;
3085       for (PetscInt i = 0; i < numFaces; i++) {
3086         for (PetscInt e = 0; e < 4; e++) {
3087           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3088           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3089 
3090           for (PetscInt d = 0; d < 3; d++) {
3091             if (!periodic || periodic[0] != DM_BOUNDARY_PERIODIC) {
3092               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) numEdges++;
3093               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) numEdges++;
3094             }
3095           }
3096         }
3097       }
3098       PetscCall(PetscMalloc1(numEdges, &edges));
3099       PetscCall(PetscMalloc1(numEdges, &edgeSets));
3100       for (PetscInt edge = 0, i = 0; i < numFaces; i++) {
3101         for (PetscInt e = 0; e < 4; e++) {
3102           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3103           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3104 
3105           for (PetscInt d = 0; d < 3; d++) {
3106             if (!periodic || periodic[d] != DM_BOUNDARY_PERIODIC) {
3107               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) {
3108                 edges[edge][0]   = ev[0];
3109                 edges[edge][1]   = ev[1];
3110                 edgeSets[edge++] = 2 * d;
3111               }
3112               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) {
3113                 edges[edge][0]   = ev[0];
3114                 edges[edge][1]   = ev[1];
3115                 edgeSets[edge++] = 2 * d + 1;
3116               }
3117             }
3118           }
3119         }
3120       }
3121     }
3122     evalFunc   = TPSEvaluate_Gyroid;
3123     normalFunc = TPSExtrudeNormalFunc_Gyroid;
3124     break;
3125   }
3126 
3127   PetscCall(DMSetDimension(dm, topoDim));
3128   if (rank == 0) PetscCall(DMPlexBuildFromCellList(dm, numFaces, numVertices, 4, cells_flat));
3129   else PetscCall(DMPlexBuildFromCellList(dm, 0, 0, 0, NULL));
3130   PetscCall(PetscFree(cells_flat));
3131   {
3132     DM idm;
3133     PetscCall(DMPlexInterpolate(dm, &idm));
3134     PetscCall(DMPlexReplace_Internal(dm, &idm));
3135   }
3136   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, vtxCoords));
3137   else PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, NULL));
3138   PetscCall(PetscFree(vtxCoords));
3139 
3140   PetscCall(DMCreateLabel(dm, "Face Sets"));
3141   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3142   for (PetscInt e = 0; e < numEdges; e++) {
3143     PetscInt        njoin;
3144     const PetscInt *join, verts[] = {numFaces + edges[e][0], numFaces + edges[e][1]};
3145     PetscCall(DMPlexGetJoin(dm, 2, verts, &njoin, &join));
3146     PetscCheck(njoin == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected unique join of vertices %" PetscInt_FMT " and %" PetscInt_FMT, edges[e][0], edges[e][1]);
3147     PetscCall(DMLabelSetValue(label, join[0], edgeSets[e]));
3148     PetscCall(DMPlexRestoreJoin(dm, 2, verts, &njoin, &join));
3149   }
3150   PetscCall(PetscFree(edges));
3151   PetscCall(PetscFree(edgeSets));
3152   if (tps_distribute) {
3153     DM               pdm = NULL;
3154     PetscPartitioner part;
3155 
3156     PetscCall(DMPlexGetPartitioner(dm, &part));
3157     PetscCall(PetscPartitionerSetFromOptions(part));
3158     PetscCall(DMPlexDistribute(dm, 0, NULL, &pdm));
3159     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
3160     // Do not auto-distribute again
3161     PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
3162   }
3163 
3164   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3165   for (PetscInt refine = 0; refine < refinements; refine++) {
3166     PetscInt     m;
3167     DM           dmf;
3168     Vec          X;
3169     PetscScalar *x;
3170     PetscCall(DMRefine(dm, MPI_COMM_NULL, &dmf));
3171     PetscCall(DMPlexReplace_Internal(dm, &dmf));
3172 
3173     PetscCall(DMGetCoordinatesLocal(dm, &X));
3174     PetscCall(VecGetLocalSize(X, &m));
3175     PetscCall(VecGetArray(X, &x));
3176     for (PetscInt i = 0; i < m; i += 3) PetscCall(TPSNearestPoint(evalFunc, &x[i]));
3177     PetscCall(VecRestoreArray(X, &x));
3178   }
3179 
3180   // Face Sets has already been propagated to new vertices during refinement; this propagates to the initial vertices.
3181   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3182   PetscCall(DMPlexLabelComplete(dm, label));
3183 
3184   if (thickness > 0) {
3185     DM              edm, cdm, ecdm;
3186     DMPlexTransform tr;
3187     const char     *prefix;
3188     PetscOptions    options;
3189     // Code from DMPlexExtrude
3190     PetscCall(DMPlexTransformCreate(PetscObjectComm((PetscObject)dm), &tr));
3191     PetscCall(DMPlexTransformSetDM(tr, dm));
3192     PetscCall(DMPlexTransformSetType(tr, DMPLEXEXTRUDE));
3193     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
3194     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)tr, prefix));
3195     PetscCall(PetscObjectGetOptions((PetscObject)dm, &options));
3196     PetscCall(PetscObjectSetOptions((PetscObject)tr, options));
3197     PetscCall(DMPlexTransformExtrudeSetLayers(tr, layers));
3198     PetscCall(DMPlexTransformExtrudeSetThickness(tr, thickness));
3199     PetscCall(DMPlexTransformExtrudeSetTensor(tr, PETSC_FALSE));
3200     PetscCall(DMPlexTransformExtrudeSetSymmetric(tr, PETSC_TRUE));
3201     PetscCall(DMPlexTransformExtrudeSetNormalFunction(tr, normalFunc));
3202     PetscCall(DMPlexTransformSetFromOptions(tr));
3203     PetscCall(PetscObjectSetOptions((PetscObject)tr, NULL));
3204     PetscCall(DMPlexTransformSetUp(tr));
3205     PetscCall(PetscObjectViewFromOptions((PetscObject)tr, NULL, "-dm_plex_tps_transform_view"));
3206     PetscCall(DMPlexTransformApply(tr, dm, &edm));
3207     PetscCall(DMCopyDisc(dm, edm));
3208     PetscCall(DMGetCoordinateDM(dm, &cdm));
3209     PetscCall(DMGetCoordinateDM(edm, &ecdm));
3210     PetscCall(DMCopyDisc(cdm, ecdm));
3211     PetscCall(DMPlexTransformCreateDiscLabels(tr, edm));
3212     PetscCall(DMPlexTransformDestroy(&tr));
3213     if (edm) {
3214       ((DM_Plex *)edm->data)->printFEM    = ((DM_Plex *)dm->data)->printFEM;
3215       ((DM_Plex *)edm->data)->printL2     = ((DM_Plex *)dm->data)->printL2;
3216       ((DM_Plex *)edm->data)->printLocate = ((DM_Plex *)dm->data)->printLocate;
3217     }
3218     PetscCall(DMPlexReplace_Internal(dm, &edm));
3219   }
3220   PetscFunctionReturn(0);
3221 }
3222 
3223 /*@
3224   DMPlexCreateTPSMesh - Create a distributed, interpolated mesh of a triply-periodic surface
3225 
3226   Collective
3227 
3228   Input Parameters:
3229 + comm   - The communicator for the `DM` object
3230 . tpstype - Type of triply-periodic surface
3231 . extent - Array of length 3 containing number of periods in each direction
3232 . periodic - array of length 3 with periodicity, or NULL for non-periodic
3233 . tps_distribute - Distribute 2D manifold mesh prior to refinement and extrusion (more scalable)
3234 . refinements - Number of factor-of-2 refinements of 2D manifold mesh
3235 . layers - Number of cell layers extruded in normal direction
3236 - thickness - Thickness in normal direction
3237 
3238   Output Parameter:
3239 . dm  - The `DM` object
3240 
3241   Level: beginner
3242 
3243   Notes:
3244   This meshes the surface of the Schwarz P or Gyroid surfaces.  Schwarz P is is the simplest member of the triply-periodic minimal surfaces.
3245   https://en.wikipedia.org/wiki/Schwarz_minimal_surface#Schwarz_P_(%22Primitive%22) and can be cut with "clean" boundaries.
3246   The Gyroid (https://en.wikipedia.org/wiki/Gyroid) is another triply-periodic minimal surface with applications in additive manufacturing; it is much more difficult to "cut" since there are no planes of symmetry.
3247   Our implementation creates a very coarse mesh of the surface and refines (by 4-way splitting) as many times as requested.
3248   On each refinement, all vertices are projected to their nearest point on the surface.
3249   This projection could readily be extended to related surfaces.
3250 
3251   The face (edge) sets for the Schwarz P surface are numbered 1(-x), 2(+x), 3(-y), 4(+y), 5(-z), 6(+z).
3252   When the mesh is refined, "Face Sets" contain the new vertices (created during refinement).  Use DMPlexLabelComplete() to propagate to coarse-level vertices.
3253 
3254   Developer Note:
3255   The Gyroid mesh does not currently mark boundary sets.
3256 
3257   References:
3258 . * - Maskery et al, Insights into the mechanical properties of several triply periodic minimal surface lattice structures made by polymer additive manufacturing, 2017.
3259     https://doi.org/10.1016/j.polymer.2017.11.049
3260 
3261 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMSetType()`, `DMCreate()`
3262 @*/
3263 PetscErrorCode DMPlexCreateTPSMesh(MPI_Comm comm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness, DM *dm)
3264 {
3265   PetscFunctionBegin;
3266   PetscCall(DMCreate(comm, dm));
3267   PetscCall(DMSetType(*dm, DMPLEX));
3268   PetscCall(DMPlexCreateTPSMesh_Internal(*dm, tpstype, extent, periodic, tps_distribute, refinements, layers, thickness));
3269   PetscFunctionReturn(0);
3270 }
3271 
3272 /*@
3273   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
3274 
3275   Collective
3276 
3277   Input Parameters:
3278 + comm    - The communicator for the `DM` object
3279 . dim     - The dimension
3280 . simplex - Use simplices, or tensor product cells
3281 - R       - The radius
3282 
3283   Output Parameter:
3284 . dm  - The `DM` object
3285 
3286   Level: beginner
3287 
3288 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBallMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3289 @*/
3290 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscReal R, DM *dm)
3291 {
3292   PetscFunctionBegin;
3293   PetscValidPointer(dm, 5);
3294   PetscCall(DMCreate(comm, dm));
3295   PetscCall(DMSetType(*dm, DMPLEX));
3296   PetscCall(DMPlexCreateSphereMesh_Internal(*dm, dim, simplex, R));
3297   PetscFunctionReturn(0);
3298 }
3299 
3300 static PetscErrorCode DMPlexCreateBallMesh_Internal(DM dm, PetscInt dim, PetscReal R)
3301 {
3302   DM      sdm, vol;
3303   DMLabel bdlabel;
3304 
3305   PetscFunctionBegin;
3306   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
3307   PetscCall(DMSetType(sdm, DMPLEX));
3308   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sdm, "bd_"));
3309   PetscCall(DMPlexCreateSphereMesh_Internal(sdm, dim - 1, PETSC_TRUE, R));
3310   PetscCall(DMSetFromOptions(sdm));
3311   PetscCall(DMViewFromOptions(sdm, NULL, "-dm_view"));
3312   PetscCall(DMPlexGenerate(sdm, NULL, PETSC_TRUE, &vol));
3313   PetscCall(DMDestroy(&sdm));
3314   PetscCall(DMPlexReplace_Internal(dm, &vol));
3315   PetscCall(DMCreateLabel(dm, "marker"));
3316   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
3317   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
3318   PetscCall(DMPlexLabelComplete(dm, bdlabel));
3319   PetscFunctionReturn(0);
3320 }
3321 
3322 /*@
3323   DMPlexCreateBallMesh - Creates a simplex mesh on the d-dimensional ball, B^d.
3324 
3325   Collective
3326 
3327   Input Parameters:
3328 + comm  - The communicator for the `DM` object
3329 . dim   - The dimension
3330 - R     - The radius
3331 
3332   Output Parameter:
3333 . dm  - The `DM` object
3334 
3335   Options Database Key:
3336 - bd_dm_refine - This will refine the surface mesh preserving the sphere geometry
3337 
3338   Level: beginner
3339 
3340 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3341 @*/
3342 PetscErrorCode DMPlexCreateBallMesh(MPI_Comm comm, PetscInt dim, PetscReal R, DM *dm)
3343 {
3344   PetscFunctionBegin;
3345   PetscCall(DMCreate(comm, dm));
3346   PetscCall(DMSetType(*dm, DMPLEX));
3347   PetscCall(DMPlexCreateBallMesh_Internal(*dm, dim, R));
3348   PetscFunctionReturn(0);
3349 }
3350 
3351 static PetscErrorCode DMPlexCreateReferenceCell_Internal(DM rdm, DMPolytopeType ct)
3352 {
3353   PetscFunctionBegin;
3354   switch (ct) {
3355   case DM_POLYTOPE_POINT: {
3356     PetscInt    numPoints[1]        = {1};
3357     PetscInt    coneSize[1]         = {0};
3358     PetscInt    cones[1]            = {0};
3359     PetscInt    coneOrientations[1] = {0};
3360     PetscScalar vertexCoords[1]     = {0.0};
3361 
3362     PetscCall(DMSetDimension(rdm, 0));
3363     PetscCall(DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3364   } break;
3365   case DM_POLYTOPE_SEGMENT: {
3366     PetscInt    numPoints[2]        = {2, 1};
3367     PetscInt    coneSize[3]         = {2, 0, 0};
3368     PetscInt    cones[2]            = {1, 2};
3369     PetscInt    coneOrientations[2] = {0, 0};
3370     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3371 
3372     PetscCall(DMSetDimension(rdm, 1));
3373     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3374   } break;
3375   case DM_POLYTOPE_POINT_PRISM_TENSOR: {
3376     PetscInt    numPoints[2]        = {2, 1};
3377     PetscInt    coneSize[3]         = {2, 0, 0};
3378     PetscInt    cones[2]            = {1, 2};
3379     PetscInt    coneOrientations[2] = {0, 0};
3380     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3381 
3382     PetscCall(DMSetDimension(rdm, 1));
3383     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3384   } break;
3385   case DM_POLYTOPE_TRIANGLE: {
3386     PetscInt    numPoints[2]        = {3, 1};
3387     PetscInt    coneSize[4]         = {3, 0, 0, 0};
3388     PetscInt    cones[3]            = {1, 2, 3};
3389     PetscInt    coneOrientations[3] = {0, 0, 0};
3390     PetscScalar vertexCoords[6]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0};
3391 
3392     PetscCall(DMSetDimension(rdm, 2));
3393     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3394   } break;
3395   case DM_POLYTOPE_QUADRILATERAL: {
3396     PetscInt    numPoints[2]        = {4, 1};
3397     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3398     PetscInt    cones[4]            = {1, 2, 3, 4};
3399     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3400     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0};
3401 
3402     PetscCall(DMSetDimension(rdm, 2));
3403     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3404   } break;
3405   case DM_POLYTOPE_SEG_PRISM_TENSOR: {
3406     PetscInt    numPoints[2]        = {4, 1};
3407     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3408     PetscInt    cones[4]            = {1, 2, 3, 4};
3409     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3410     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0};
3411 
3412     PetscCall(DMSetDimension(rdm, 2));
3413     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3414   } break;
3415   case DM_POLYTOPE_TETRAHEDRON: {
3416     PetscInt    numPoints[2]        = {4, 1};
3417     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3418     PetscInt    cones[4]            = {1, 2, 3, 4};
3419     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3420     PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0};
3421 
3422     PetscCall(DMSetDimension(rdm, 3));
3423     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3424   } break;
3425   case DM_POLYTOPE_HEXAHEDRON: {
3426     PetscInt    numPoints[2]        = {8, 1};
3427     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3428     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3429     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3430     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3431 
3432     PetscCall(DMSetDimension(rdm, 3));
3433     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3434   } break;
3435   case DM_POLYTOPE_TRI_PRISM: {
3436     PetscInt    numPoints[2]        = {6, 1};
3437     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3438     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3439     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3440     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3441 
3442     PetscCall(DMSetDimension(rdm, 3));
3443     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3444   } break;
3445   case DM_POLYTOPE_TRI_PRISM_TENSOR: {
3446     PetscInt    numPoints[2]        = {6, 1};
3447     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3448     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3449     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3450     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3451 
3452     PetscCall(DMSetDimension(rdm, 3));
3453     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3454   } break;
3455   case DM_POLYTOPE_QUAD_PRISM_TENSOR: {
3456     PetscInt    numPoints[2]        = {8, 1};
3457     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3458     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3459     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3460     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3461 
3462     PetscCall(DMSetDimension(rdm, 3));
3463     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3464   } break;
3465   case DM_POLYTOPE_PYRAMID: {
3466     PetscInt    numPoints[2]        = {5, 1};
3467     PetscInt    coneSize[6]         = {5, 0, 0, 0, 0, 0};
3468     PetscInt    cones[5]            = {1, 2, 3, 4, 5};
3469     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3470     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, 0.0, 0.0, 1.0};
3471 
3472     PetscCall(DMSetDimension(rdm, 3));
3473     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3474   } break;
3475   default:
3476     SETERRQ(PetscObjectComm((PetscObject)rdm), PETSC_ERR_ARG_WRONG, "Cannot create reference cell for cell type %s", DMPolytopeTypes[ct]);
3477   }
3478   {
3479     PetscInt Nv, v;
3480 
3481     /* Must create the celltype label here so that we do not automatically try to compute the types */
3482     PetscCall(DMCreateLabel(rdm, "celltype"));
3483     PetscCall(DMPlexSetCellType(rdm, 0, ct));
3484     PetscCall(DMPlexGetChart(rdm, NULL, &Nv));
3485     for (v = 1; v < Nv; ++v) PetscCall(DMPlexSetCellType(rdm, v, DM_POLYTOPE_POINT));
3486   }
3487   PetscCall(DMPlexInterpolateInPlace_Internal(rdm));
3488   PetscCall(PetscObjectSetName((PetscObject)rdm, DMPolytopeTypes[ct]));
3489   PetscFunctionReturn(0);
3490 }
3491 
3492 /*@
3493   DMPlexCreateReferenceCell - Create a `DMPLEX` with the appropriate FEM reference cell
3494 
3495   Collective
3496 
3497   Input Parameters:
3498 + comm - The communicator
3499 - ct   - The cell type of the reference cell
3500 
3501   Output Parameter:
3502 . refdm - The reference cell
3503 
3504   Level: intermediate
3505 
3506 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateReferenceCell()`, `DMPlexCreateBoxMesh()`
3507 @*/
3508 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, DMPolytopeType ct, DM *refdm)
3509 {
3510   PetscFunctionBegin;
3511   PetscCall(DMCreate(comm, refdm));
3512   PetscCall(DMSetType(*refdm, DMPLEX));
3513   PetscCall(DMPlexCreateReferenceCell_Internal(*refdm, ct));
3514   PetscFunctionReturn(0);
3515 }
3516 
3517 static PetscErrorCode DMPlexCreateBoundaryLabel_Private(DM dm, const char name[])
3518 {
3519   DM        plex;
3520   DMLabel   label;
3521   PetscBool hasLabel;
3522 
3523   PetscFunctionBegin;
3524   PetscCall(DMHasLabel(dm, name, &hasLabel));
3525   if (hasLabel) PetscFunctionReturn(0);
3526   PetscCall(DMCreateLabel(dm, name));
3527   PetscCall(DMGetLabel(dm, name, &label));
3528   PetscCall(DMConvert(dm, DMPLEX, &plex));
3529   PetscCall(DMPlexMarkBoundaryFaces(plex, 1, label));
3530   PetscCall(DMPlexLabelComplete(plex, label));
3531   PetscCall(DMDestroy(&plex));
3532   PetscFunctionReturn(0);
3533 }
3534 
3535 /*
3536   We use the last coordinate as the radius, the inner radius is lower[dim-1] and the outer radius is upper[dim-1]. Then we map the first coordinate around the circle.
3537 
3538     (x, y) -> (r, theta) = (x[1], (x[0] - lower[0]) * 2\pi/(upper[0] - lower[0]))
3539 */
3540 static void boxToAnnulus(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
3541 {
3542   const PetscReal low = PetscRealPart(constants[0]);
3543   const PetscReal upp = PetscRealPart(constants[1]);
3544   const PetscReal r   = PetscRealPart(u[1]);
3545   const PetscReal th  = 2. * PETSC_PI * (PetscRealPart(u[0]) - low) / (upp - low);
3546 
3547   f0[0] = r * PetscCosReal(th);
3548   f0[1] = r * PetscSinReal(th);
3549 }
3550 
3551 const char *const DMPlexShapes[] = {"box", "box_surface", "ball", "sphere", "cylinder", "schwarz_p", "gyroid", "doublet", "annulus", "unknown", "DMPlexShape", "DM_SHAPE_", NULL};
3552 
3553 static PetscErrorCode DMPlexCreateFromOptions_Internal(PetscOptionItems *PetscOptionsObject, PetscBool *useCoordSpace, DM dm)
3554 {
3555   DMPlexShape    shape   = DM_SHAPE_BOX;
3556   DMPolytopeType cell    = DM_POLYTOPE_TRIANGLE;
3557   PetscInt       dim     = 2;
3558   PetscBool      simplex = PETSC_TRUE, interpolate = PETSC_TRUE, adjCone = PETSC_FALSE, adjClosure = PETSC_TRUE, refDomain = PETSC_FALSE;
3559   PetscBool      flg, flg2, fflg, bdfflg, nameflg;
3560   MPI_Comm       comm;
3561   char           filename[PETSC_MAX_PATH_LEN]   = "<unspecified>";
3562   char           bdFilename[PETSC_MAX_PATH_LEN] = "<unspecified>";
3563   char           plexname[PETSC_MAX_PATH_LEN]   = "";
3564 
3565   PetscFunctionBegin;
3566   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3567   /* TODO Turn this into a registration interface */
3568   PetscCall(PetscOptionsString("-dm_plex_filename", "File containing a mesh", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &fflg));
3569   PetscCall(PetscOptionsString("-dm_plex_boundary_filename", "File containing a mesh boundary", "DMPlexCreateFromFile", bdFilename, bdFilename, sizeof(bdFilename), &bdfflg));
3570   PetscCall(PetscOptionsString("-dm_plex_name", "Name of the mesh in the file", "DMPlexCreateFromFile", plexname, plexname, sizeof(plexname), &nameflg));
3571   PetscCall(PetscOptionsEnum("-dm_plex_cell", "Cell shape", "", DMPolytopeTypes, (PetscEnum)cell, (PetscEnum *)&cell, NULL));
3572   PetscCall(PetscOptionsBool("-dm_plex_reference_cell_domain", "Use a reference cell domain", "", refDomain, &refDomain, NULL));
3573   PetscCall(PetscOptionsEnum("-dm_plex_shape", "Shape for built-in mesh", "", DMPlexShapes, (PetscEnum)shape, (PetscEnum *)&shape, &flg));
3574   PetscCall(PetscOptionsBoundedInt("-dm_plex_dim", "Topological dimension of the mesh", "DMGetDimension", dim, &dim, &flg, 0));
3575   PetscCheck(!(dim < 0) && !(dim > 3), comm, PETSC_ERR_ARG_OUTOFRANGE, "Dimension %" PetscInt_FMT " should be in [1, 3]", dim);
3576   PetscCall(PetscOptionsBool("-dm_plex_simplex", "Mesh cell shape", "", simplex, &simplex, &flg));
3577   PetscCall(PetscOptionsBool("-dm_plex_interpolate", "Flag to create edges and faces automatically", "", interpolate, &interpolate, &flg));
3578   PetscCall(PetscOptionsBool("-dm_plex_adj_cone", "Set adjacency direction", "DMSetBasicAdjacency", adjCone, &adjCone, &flg));
3579   PetscCall(PetscOptionsBool("-dm_plex_adj_closure", "Set adjacency size", "DMSetBasicAdjacency", adjClosure, &adjClosure, &flg2));
3580   if (flg || flg2) PetscCall(DMSetBasicAdjacency(dm, adjCone, adjClosure));
3581 
3582   switch (cell) {
3583   case DM_POLYTOPE_POINT:
3584   case DM_POLYTOPE_SEGMENT:
3585   case DM_POLYTOPE_POINT_PRISM_TENSOR:
3586   case DM_POLYTOPE_TRIANGLE:
3587   case DM_POLYTOPE_QUADRILATERAL:
3588   case DM_POLYTOPE_TETRAHEDRON:
3589   case DM_POLYTOPE_HEXAHEDRON:
3590     *useCoordSpace = PETSC_TRUE;
3591     break;
3592   default:
3593     *useCoordSpace = PETSC_FALSE;
3594     break;
3595   }
3596 
3597   if (fflg) {
3598     DM dmnew;
3599 
3600     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), filename, plexname, interpolate, &dmnew));
3601     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3602     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3603   } else if (refDomain) {
3604     PetscCall(DMPlexCreateReferenceCell_Internal(dm, cell));
3605   } else if (bdfflg) {
3606     DM bdm, dmnew;
3607 
3608     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), bdFilename, plexname, interpolate, &bdm));
3609     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)bdm, "bd_"));
3610     PetscCall(DMSetFromOptions(bdm));
3611     PetscCall(DMPlexGenerate(bdm, NULL, interpolate, &dmnew));
3612     PetscCall(DMDestroy(&bdm));
3613     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3614     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3615   } else {
3616     PetscCall(PetscObjectSetName((PetscObject)dm, DMPlexShapes[shape]));
3617     switch (shape) {
3618     case DM_SHAPE_BOX:
3619     case DM_SHAPE_ANNULUS: {
3620       PetscInt       faces[3]  = {0, 0, 0};
3621       PetscReal      lower[3]  = {0, 0, 0};
3622       PetscReal      upper[3]  = {1, 1, 1};
3623       DMBoundaryType bdt[3]    = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3624       PetscBool      isAnnular = shape == DM_SHAPE_ANNULUS ? PETSC_TRUE : PETSC_FALSE;
3625       PetscInt       i, n;
3626 
3627       n = dim;
3628       for (i = 0; i < dim; ++i) faces[i] = (dim == 1 ? 1 : 4 - dim);
3629       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3630       n = 3;
3631       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3632       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3633       n = 3;
3634       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3635       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3636       n = 3;
3637       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
3638       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3639 
3640       PetscCheck(!isAnnular || dim == 2, comm, PETSC_ERR_ARG_OUTOFRANGE, "Only two dimensional annuli have been implemented");
3641       if (isAnnular)
3642         for (i = 0; i < dim - 1; ++i) bdt[i] = DM_BOUNDARY_PERIODIC;
3643 
3644       switch (cell) {
3645       case DM_POLYTOPE_TRI_PRISM_TENSOR:
3646         PetscCall(DMPlexCreateWedgeBoxMesh_Internal(dm, faces, lower, upper, bdt));
3647         if (!interpolate) {
3648           DM udm;
3649 
3650           PetscCall(DMPlexUninterpolate(dm, &udm));
3651           PetscCall(DMPlexReplace_Internal(dm, &udm));
3652         }
3653         break;
3654       default:
3655         PetscCall(DMPlexCreateBoxMesh_Internal(dm, dim, simplex, faces, lower, upper, bdt, interpolate));
3656         break;
3657       }
3658       if (isAnnular) {
3659         DM          cdm;
3660         PetscDS     cds;
3661         PetscScalar bounds[2] = {lower[0], upper[0]};
3662 
3663         // Fix coordinates for annular region
3664         PetscCall(DMSetPeriodicity(dm, NULL, NULL, NULL));
3665         PetscCall(DMSetCellCoordinatesLocal(dm, NULL));
3666         PetscCall(DMSetCellCoordinates(dm, NULL));
3667         PetscCall(DMPlexCreateCoordinateSpace(dm, 1, NULL));
3668         PetscCall(DMGetCoordinateDM(dm, &cdm));
3669         PetscCall(DMGetDS(cdm, &cds));
3670         PetscCall(PetscDSSetConstants(cds, 2, bounds));
3671         PetscCall(DMPlexRemapGeometry(dm, 0.0, boxToAnnulus));
3672       }
3673     } break;
3674     case DM_SHAPE_BOX_SURFACE: {
3675       PetscInt  faces[3] = {0, 0, 0};
3676       PetscReal lower[3] = {0, 0, 0};
3677       PetscReal upper[3] = {1, 1, 1};
3678       PetscInt  i, n;
3679 
3680       n = dim + 1;
3681       for (i = 0; i < dim + 1; ++i) faces[i] = (dim + 1 == 1 ? 1 : 4 - (dim + 1));
3682       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3683       n = 3;
3684       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3685       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3686       n = 3;
3687       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3688       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3689       PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(dm, dim + 1, faces, lower, upper, interpolate));
3690     } break;
3691     case DM_SHAPE_SPHERE: {
3692       PetscReal R = 1.0;
3693 
3694       PetscCall(PetscOptionsReal("-dm_plex_sphere_radius", "Radius of the sphere", "", R, &R, &flg));
3695       PetscCall(DMPlexCreateSphereMesh_Internal(dm, dim, simplex, R));
3696     } break;
3697     case DM_SHAPE_BALL: {
3698       PetscReal R = 1.0;
3699 
3700       PetscCall(PetscOptionsReal("-dm_plex_ball_radius", "Radius of the ball", "", R, &R, &flg));
3701       PetscCall(DMPlexCreateBallMesh_Internal(dm, dim, R));
3702     } break;
3703     case DM_SHAPE_CYLINDER: {
3704       DMBoundaryType bdt = DM_BOUNDARY_NONE;
3705       PetscInt       Nw  = 6;
3706 
3707       PetscCall(PetscOptionsEnum("-dm_plex_cylinder_bd", "Boundary type in the z direction", "", DMBoundaryTypes, (PetscEnum)bdt, (PetscEnum *)&bdt, NULL));
3708       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_wedges", "Number of wedges around the cylinder", "", Nw, &Nw, NULL));
3709       switch (cell) {
3710       case DM_POLYTOPE_TRI_PRISM_TENSOR:
3711         PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(dm, Nw, interpolate));
3712         break;
3713       default:
3714         PetscCall(DMPlexCreateHexCylinderMesh_Internal(dm, bdt));
3715         break;
3716       }
3717     } break;
3718     case DM_SHAPE_SCHWARZ_P: // fallthrough
3719     case DM_SHAPE_GYROID: {
3720       PetscInt       extent[3] = {1, 1, 1}, refine = 0, layers = 0, three;
3721       PetscReal      thickness   = 0.;
3722       DMBoundaryType periodic[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3723       DMPlexTPSType  tps_type    = shape == DM_SHAPE_SCHWARZ_P ? DMPLEX_TPS_SCHWARZ_P : DMPLEX_TPS_GYROID;
3724       PetscBool      tps_distribute;
3725       PetscCall(PetscOptionsIntArray("-dm_plex_tps_extent", "Number of replicas for each of three dimensions", NULL, extent, (three = 3, &three), NULL));
3726       PetscCall(PetscOptionsInt("-dm_plex_tps_refine", "Number of refinements", NULL, refine, &refine, NULL));
3727       PetscCall(PetscOptionsEnumArray("-dm_plex_tps_periodic", "Periodicity in each of three dimensions", NULL, DMBoundaryTypes, (PetscEnum *)periodic, (three = 3, &three), NULL));
3728       PetscCall(PetscOptionsInt("-dm_plex_tps_layers", "Number of layers in volumetric extrusion (or zero to not extrude)", NULL, layers, &layers, NULL));
3729       PetscCall(PetscOptionsReal("-dm_plex_tps_thickness", "Thickness of volumetric extrusion", NULL, thickness, &thickness, NULL));
3730       PetscCall(DMPlexDistributeGetDefault(dm, &tps_distribute));
3731       PetscCall(PetscOptionsBool("-dm_plex_tps_distribute", "Distribute the 2D mesh prior to refinement and extrusion", NULL, tps_distribute, &tps_distribute, NULL));
3732       PetscCall(DMPlexCreateTPSMesh_Internal(dm, tps_type, extent, periodic, tps_distribute, refine, layers, thickness));
3733     } break;
3734     case DM_SHAPE_DOUBLET: {
3735       DM        dmnew;
3736       PetscReal rl = 0.0;
3737 
3738       PetscCall(PetscOptionsReal("-dm_plex_doublet_refinementlimit", "Refinement limit", NULL, rl, &rl, NULL));
3739       PetscCall(DMPlexCreateDoublet(PetscObjectComm((PetscObject)dm), dim, simplex, interpolate, rl, &dmnew));
3740       PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3741       PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3742     } break;
3743     default:
3744       SETERRQ(comm, PETSC_ERR_SUP, "Domain shape %s is unsupported", DMPlexShapes[shape]);
3745     }
3746   }
3747   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3748   if (!((PetscObject)dm)->name && nameflg) PetscCall(PetscObjectSetName((PetscObject)dm, plexname));
3749   PetscFunctionReturn(0);
3750 }
3751 
3752 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
3753 {
3754   DM_Plex  *mesh = (DM_Plex *)dm->data;
3755   PetscBool flg, flg2;
3756   char      bdLabel[PETSC_MAX_PATH_LEN];
3757 
3758   PetscFunctionBegin;
3759   /* Handle viewing */
3760   PetscCall(PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL));
3761   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fem", "Debug output level all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL, 0));
3762   PetscCall(PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL));
3763   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL, 0));
3764   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_locate", "Debug output level all point location computations", "DMLocatePoints", 0, &mesh->printLocate, NULL, 0));
3765   PetscCall(DMMonitorSetFromOptions(dm, "-dm_plex_monitor_throughput", "Monitor the simulation throughput", "DMPlexMonitorThroughput", DMPlexMonitorThroughput, NULL, &flg));
3766   if (flg) PetscCall(PetscLogDefaultBegin());
3767   /* Labeling */
3768   PetscCall(PetscOptionsString("-dm_plex_boundary_label", "Label to mark the mesh boundary", "", bdLabel, bdLabel, sizeof(bdLabel), &flg));
3769   if (flg) PetscCall(DMPlexCreateBoundaryLabel_Private(dm, bdLabel));
3770   /* Point Location */
3771   PetscCall(PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL));
3772   /* Partitioning and distribution */
3773   PetscCall(PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL));
3774   /* Generation and remeshing */
3775   PetscCall(PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL));
3776   /* Projection behavior */
3777   PetscCall(PetscOptionsBoundedInt("-dm_plex_max_projection_height", "Maximum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL, 0));
3778   PetscCall(PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL));
3779   /* Checking structure */
3780   {
3781     PetscBool all = PETSC_FALSE;
3782 
3783     PetscCall(PetscOptionsBool("-dm_plex_check_all", "Perform all basic checks", "DMPlexCheck", PETSC_FALSE, &all, NULL));
3784     if (all) {
3785       PetscCall(DMPlexCheck(dm));
3786     } else {
3787       PetscCall(PetscOptionsBool("-dm_plex_check_symmetry", "Check that the adjacency information in the mesh is symmetric", "DMPlexCheckSymmetry", PETSC_FALSE, &flg, &flg2));
3788       if (flg && flg2) PetscCall(DMPlexCheckSymmetry(dm));
3789       PetscCall(PetscOptionsBool("-dm_plex_check_skeleton", "Check that each cell has the correct number of vertices (only for homogeneous simplex or tensor meshes)", "DMPlexCheckSkeleton", PETSC_FALSE, &flg, &flg2));
3790       if (flg && flg2) PetscCall(DMPlexCheckSkeleton(dm, 0));
3791       PetscCall(PetscOptionsBool("-dm_plex_check_faces", "Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type", "DMPlexCheckFaces", PETSC_FALSE, &flg, &flg2));
3792       if (flg && flg2) PetscCall(DMPlexCheckFaces(dm, 0));
3793       PetscCall(PetscOptionsBool("-dm_plex_check_geometry", "Check that cells have positive volume", "DMPlexCheckGeometry", PETSC_FALSE, &flg, &flg2));
3794       if (flg && flg2) PetscCall(DMPlexCheckGeometry(dm));
3795       PetscCall(PetscOptionsBool("-dm_plex_check_pointsf", "Check some necessary conditions for PointSF", "DMPlexCheckPointSF", PETSC_FALSE, &flg, &flg2));
3796       if (flg && flg2) PetscCall(DMPlexCheckPointSF(dm, NULL, PETSC_FALSE));
3797       PetscCall(PetscOptionsBool("-dm_plex_check_interface_cones", "Check points on inter-partition interfaces have conforming order of cone points", "DMPlexCheckInterfaceCones", PETSC_FALSE, &flg, &flg2));
3798       if (flg && flg2) PetscCall(DMPlexCheckInterfaceCones(dm));
3799     }
3800     PetscCall(PetscOptionsBool("-dm_plex_check_cell_shape", "Check cell shape", "DMPlexCheckCellShape", PETSC_FALSE, &flg, &flg2));
3801     if (flg && flg2) PetscCall(DMPlexCheckCellShape(dm, PETSC_TRUE, PETSC_DETERMINE));
3802   }
3803   {
3804     PetscReal scale = 1.0;
3805 
3806     PetscCall(PetscOptionsReal("-dm_plex_scale", "Scale factor for mesh coordinates", "DMPlexScale", scale, &scale, &flg));
3807     if (flg) {
3808       Vec coordinates, coordinatesLocal;
3809 
3810       PetscCall(DMGetCoordinates(dm, &coordinates));
3811       PetscCall(DMGetCoordinatesLocal(dm, &coordinatesLocal));
3812       PetscCall(VecScale(coordinates, scale));
3813       PetscCall(VecScale(coordinatesLocal, scale));
3814     }
3815   }
3816   PetscCall(PetscPartitionerSetFromOptions(mesh->partitioner));
3817   PetscFunctionReturn(0);
3818 }
3819 
3820 PetscErrorCode DMSetFromOptions_Overlap_Plex(DM dm, PetscOptionItems *PetscOptionsObject, PetscInt *overlap)
3821 {
3822   PetscInt  numOvLabels = 16, numOvExLabels = 16;
3823   char     *ovLabelNames[16], *ovExLabelNames[16];
3824   PetscInt  numOvValues = 16, numOvExValues = 16, l;
3825   PetscBool flg;
3826 
3827   PetscFunctionBegin;
3828   PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", *overlap, overlap, NULL, 0));
3829   PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_labels", "List of overlap label names", "DMPlexDistribute", ovLabelNames, &numOvLabels, &flg));
3830   if (!flg) numOvLabels = 0;
3831   if (numOvLabels) {
3832     ((DM_Plex *)dm->data)->numOvLabels = numOvLabels;
3833     for (l = 0; l < numOvLabels; ++l) {
3834       PetscCall(DMGetLabel(dm, ovLabelNames[l], &((DM_Plex *)dm->data)->ovLabels[l]));
3835       PetscCheck(((DM_Plex *)dm->data)->ovLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovLabelNames[l]);
3836       PetscCall(PetscFree(ovLabelNames[l]));
3837     }
3838     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_values", "List of overlap label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovValues, &numOvValues, &flg));
3839     if (!flg) numOvValues = 0;
3840     PetscCheck(numOvLabels == numOvValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvLabels, numOvValues);
3841 
3842     PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_exclude_labels", "List of overlap exclude label names", "DMPlexDistribute", ovExLabelNames, &numOvExLabels, &flg));
3843     if (!flg) numOvExLabels = 0;
3844     ((DM_Plex *)dm->data)->numOvExLabels = numOvExLabels;
3845     for (l = 0; l < numOvExLabels; ++l) {
3846       PetscCall(DMGetLabel(dm, ovExLabelNames[l], &((DM_Plex *)dm->data)->ovExLabels[l]));
3847       PetscCheck(((DM_Plex *)dm->data)->ovExLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovExLabelNames[l]);
3848       PetscCall(PetscFree(ovExLabelNames[l]));
3849     }
3850     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_exclude_values", "List of overlap exclude label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovExValues, &numOvExValues, &flg));
3851     if (!flg) numOvExValues = 0;
3852     PetscCheck(numOvExLabels == numOvExValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of exclude labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvExLabels, numOvExValues);
3853   }
3854   PetscFunctionReturn(0);
3855 }
3856 
3857 static PetscErrorCode DMSetFromOptions_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
3858 {
3859   PetscFunctionList        ordlist;
3860   char                     oname[256];
3861   PetscReal                volume    = -1.0;
3862   PetscInt                 prerefine = 0, refine = 0, r, coarsen = 0, overlap = 0, extLayers = 0, dim;
3863   PetscBool                uniformOrig, created = PETSC_FALSE, uniform = PETSC_TRUE, distribute, interpolate = PETSC_TRUE, coordSpace = PETSC_TRUE, remap = PETSC_TRUE, ghostCells = PETSC_FALSE, isHierarchy, ignoreModel = PETSC_FALSE, flg;
3864   DMPlexReorderDefaultFlag reorder;
3865 
3866   PetscFunctionBegin;
3867   PetscOptionsHeadBegin(PetscOptionsObject, "DMPlex Options");
3868   /* Handle automatic creation */
3869   PetscCall(DMGetDimension(dm, &dim));
3870   if (dim < 0) {
3871     PetscCall(DMPlexCreateFromOptions_Internal(PetscOptionsObject, &coordSpace, dm));
3872     created = PETSC_TRUE;
3873   }
3874   PetscCall(DMGetDimension(dm, &dim));
3875   /* Handle interpolation before distribution */
3876   PetscCall(PetscOptionsBool("-dm_plex_interpolate_pre", "Flag to interpolate mesh before distribution", "", interpolate, &interpolate, &flg));
3877   if (flg) {
3878     DMPlexInterpolatedFlag interpolated;
3879 
3880     PetscCall(DMPlexIsInterpolated(dm, &interpolated));
3881     if (interpolated == DMPLEX_INTERPOLATED_FULL && !interpolate) {
3882       DM udm;
3883 
3884       PetscCall(DMPlexUninterpolate(dm, &udm));
3885       PetscCall(DMPlexReplace_Internal(dm, &udm));
3886     } else if (interpolated != DMPLEX_INTERPOLATED_FULL && interpolate) {
3887       DM idm;
3888 
3889       PetscCall(DMPlexInterpolate(dm, &idm));
3890       PetscCall(DMPlexReplace_Internal(dm, &idm));
3891     }
3892   }
3893   /* Handle DMPlex refinement before distribution */
3894   PetscCall(PetscOptionsBool("-dm_refine_ignore_model", "Flag to ignore the geometry model when refining", "DMCreate", ignoreModel, &ignoreModel, &flg));
3895   if (flg) ((DM_Plex *)dm->data)->ignoreModel = ignoreModel;
3896   PetscCall(DMPlexGetRefinementUniform(dm, &uniformOrig));
3897   PetscCall(PetscOptionsBoundedInt("-dm_refine_pre", "The number of refinements before distribution", "DMCreate", prerefine, &prerefine, NULL, 0));
3898   PetscCall(PetscOptionsBool("-dm_refine_remap_pre", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
3899   PetscCall(PetscOptionsBool("-dm_refine_uniform_pre", "Flag for uniform refinement before distribution", "DMCreate", uniform, &uniform, &flg));
3900   if (flg) PetscCall(DMPlexSetRefinementUniform(dm, uniform));
3901   PetscCall(PetscOptionsReal("-dm_refine_volume_limit_pre", "The maximum cell volume after refinement before distribution", "DMCreate", volume, &volume, &flg));
3902   if (flg) {
3903     PetscCall(DMPlexSetRefinementUniform(dm, PETSC_FALSE));
3904     PetscCall(DMPlexSetRefinementLimit(dm, volume));
3905     prerefine = PetscMax(prerefine, 1);
3906   }
3907   for (r = 0; r < prerefine; ++r) {
3908     DM             rdm;
3909     PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
3910 
3911     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3912     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
3913     PetscCall(DMPlexReplace_Internal(dm, &rdm));
3914     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3915     if (coordFunc && remap) {
3916       PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
3917       ((DM_Plex *)dm->data)->coordFunc = coordFunc;
3918     }
3919   }
3920   PetscCall(DMPlexSetRefinementUniform(dm, uniformOrig));
3921   /* Handle DMPlex extrusion before distribution */
3922   PetscCall(PetscOptionsBoundedInt("-dm_extrude", "The number of layers to extrude", "", extLayers, &extLayers, NULL, 0));
3923   if (extLayers) {
3924     DM edm;
3925 
3926     PetscCall(DMExtrude(dm, extLayers, &edm));
3927     PetscCall(DMPlexReplace_Internal(dm, &edm));
3928     ((DM_Plex *)dm->data)->coordFunc = NULL;
3929     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3930     extLayers = 0;
3931   }
3932   /* Handle DMPlex reordering before distribution */
3933   PetscCall(DMPlexReorderGetDefault(dm, &reorder));
3934   PetscCall(MatGetOrderingList(&ordlist));
3935   PetscCall(PetscStrncpy(oname, MATORDERINGNATURAL, sizeof(oname)));
3936   PetscCall(PetscOptionsFList("-dm_plex_reorder", "Set mesh reordering type", "DMPlexGetOrdering", ordlist, MATORDERINGNATURAL, oname, sizeof(oname), &flg));
3937   if (reorder == DMPLEX_REORDER_DEFAULT_TRUE || flg) {
3938     DM pdm;
3939     IS perm;
3940 
3941     PetscCall(DMPlexGetOrdering(dm, oname, NULL, &perm));
3942     PetscCall(DMPlexPermute(dm, perm, &pdm));
3943     PetscCall(ISDestroy(&perm));
3944     PetscCall(DMPlexReplace_Internal(dm, &pdm));
3945     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3946   }
3947   /* Handle DMPlex distribution */
3948   PetscCall(DMPlexDistributeGetDefault(dm, &distribute));
3949   PetscCall(PetscOptionsBool("-dm_distribute", "Flag to redistribute a mesh among processes", "DMPlexDistribute", distribute, &distribute, NULL));
3950   PetscCall(DMSetFromOptions_Overlap_Plex(dm, PetscOptionsObject, &overlap));
3951   if (distribute) {
3952     DM               pdm = NULL;
3953     PetscPartitioner part;
3954 
3955     PetscCall(DMPlexGetPartitioner(dm, &part));
3956     PetscCall(PetscPartitionerSetFromOptions(part));
3957     PetscCall(DMPlexDistribute(dm, overlap, NULL, &pdm));
3958     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
3959   }
3960   /* Create coordinate space */
3961   if (created) {
3962     DM_Plex  *mesh   = (DM_Plex *)dm->data;
3963     PetscInt  degree = 1;
3964     PetscBool flg;
3965 
3966     PetscCall(PetscOptionsBool("-dm_coord_space", "Use an FEM space for coordinates", "", coordSpace, &coordSpace, &flg));
3967     PetscCall(PetscOptionsInt("-dm_coord_petscspace_degree", "FEM degree for coordinate space", "", degree, &degree, NULL));
3968     if (coordSpace) PetscCall(DMPlexCreateCoordinateSpace(dm, degree, mesh->coordFunc));
3969     if (flg && !coordSpace) {
3970       DM           cdm;
3971       PetscDS      cds;
3972       PetscObject  obj;
3973       PetscClassId id;
3974 
3975       PetscCall(DMGetCoordinateDM(dm, &cdm));
3976       PetscCall(DMGetDS(cdm, &cds));
3977       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
3978       PetscCall(PetscObjectGetClassId(obj, &id));
3979       if (id == PETSCFE_CLASSID) {
3980         PetscContainer dummy;
3981 
3982         PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &dummy));
3983         PetscCall(PetscObjectSetName((PetscObject)dummy, "coordinates"));
3984         PetscCall(DMSetField(cdm, 0, NULL, (PetscObject)dummy));
3985         PetscCall(PetscContainerDestroy(&dummy));
3986         PetscCall(DMClearDS(cdm));
3987       }
3988       mesh->coordFunc = NULL;
3989     }
3990     PetscCall(PetscOptionsBool("-dm_sparse_localize", "Localize only necessary cells", "", dm->sparseLocalize, &dm->sparseLocalize, &flg));
3991     PetscCall(DMLocalizeCoordinates(dm));
3992   }
3993   /* Handle DMPlex refinement */
3994   remap = PETSC_TRUE;
3995   PetscCall(PetscOptionsBoundedInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL, 0));
3996   PetscCall(PetscOptionsBool("-dm_refine_remap", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
3997   PetscCall(PetscOptionsBoundedInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy, 0));
3998   if (refine) PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3999   if (refine && isHierarchy) {
4000     DM *dms, coarseDM;
4001 
4002     PetscCall(DMGetCoarseDM(dm, &coarseDM));
4003     PetscCall(PetscObjectReference((PetscObject)coarseDM));
4004     PetscCall(PetscMalloc1(refine, &dms));
4005     PetscCall(DMRefineHierarchy(dm, refine, dms));
4006     /* Total hack since we do not pass in a pointer */
4007     PetscCall(DMPlexSwap_Static(dm, dms[refine - 1]));
4008     if (refine == 1) {
4009       PetscCall(DMSetCoarseDM(dm, dms[0]));
4010       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4011     } else {
4012       PetscCall(DMSetCoarseDM(dm, dms[refine - 2]));
4013       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4014       PetscCall(DMSetCoarseDM(dms[0], dms[refine - 1]));
4015       PetscCall(DMPlexSetRegularRefinement(dms[0], PETSC_TRUE));
4016     }
4017     PetscCall(DMSetCoarseDM(dms[refine - 1], coarseDM));
4018     PetscCall(PetscObjectDereference((PetscObject)coarseDM));
4019     /* Free DMs */
4020     for (r = 0; r < refine; ++r) {
4021       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4022       PetscCall(DMDestroy(&dms[r]));
4023     }
4024     PetscCall(PetscFree(dms));
4025   } else {
4026     for (r = 0; r < refine; ++r) {
4027       DM             rdm;
4028       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4029 
4030       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4031       PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
4032       /* Total hack since we do not pass in a pointer */
4033       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4034       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4035       if (coordFunc && remap) {
4036         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4037         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4038       }
4039     }
4040   }
4041   /* Handle DMPlex coarsening */
4042   PetscCall(PetscOptionsBoundedInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL, 0));
4043   PetscCall(PetscOptionsBoundedInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy, 0));
4044   if (coarsen && isHierarchy) {
4045     DM *dms;
4046 
4047     PetscCall(PetscMalloc1(coarsen, &dms));
4048     PetscCall(DMCoarsenHierarchy(dm, coarsen, dms));
4049     /* Free DMs */
4050     for (r = 0; r < coarsen; ++r) {
4051       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4052       PetscCall(DMDestroy(&dms[r]));
4053     }
4054     PetscCall(PetscFree(dms));
4055   } else {
4056     for (r = 0; r < coarsen; ++r) {
4057       DM             cdm;
4058       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4059 
4060       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4061       PetscCall(DMCoarsen(dm, PetscObjectComm((PetscObject)dm), &cdm));
4062       /* Total hack since we do not pass in a pointer */
4063       PetscCall(DMPlexReplace_Internal(dm, &cdm));
4064       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4065       if (coordFunc) {
4066         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4067         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4068       }
4069     }
4070   }
4071   /* Handle ghost cells */
4072   PetscCall(PetscOptionsBool("-dm_plex_create_fv_ghost_cells", "Flag to create finite volume ghost cells on the boundary", "DMCreate", ghostCells, &ghostCells, NULL));
4073   if (ghostCells) {
4074     DM   gdm;
4075     char lname[PETSC_MAX_PATH_LEN];
4076 
4077     lname[0] = '\0';
4078     PetscCall(PetscOptionsString("-dm_plex_fv_ghost_cells_label", "Label name for ghost cells boundary", "DMCreate", lname, lname, sizeof(lname), &flg));
4079     PetscCall(DMPlexConstructGhostCells(dm, flg ? lname : NULL, NULL, &gdm));
4080     PetscCall(DMPlexReplace_Internal(dm, &gdm));
4081   }
4082   /* Handle 1D order */
4083   if (reorder != DMPLEX_REORDER_DEFAULT_FALSE && dim == 1) {
4084     DM           cdm, rdm;
4085     PetscDS      cds;
4086     PetscObject  obj;
4087     PetscClassId id = PETSC_OBJECT_CLASSID;
4088     IS           perm;
4089     PetscInt     Nf;
4090     PetscBool    distributed;
4091 
4092     PetscCall(DMPlexIsDistributed(dm, &distributed));
4093     PetscCall(DMGetCoordinateDM(dm, &cdm));
4094     PetscCall(DMGetDS(cdm, &cds));
4095     PetscCall(PetscDSGetNumFields(cds, &Nf));
4096     if (Nf) {
4097       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
4098       PetscCall(PetscObjectGetClassId(obj, &id));
4099     }
4100     if (!distributed && id != PETSCFE_CLASSID) {
4101       PetscCall(DMPlexGetOrdering1D(dm, &perm));
4102       PetscCall(DMPlexPermute(dm, perm, &rdm));
4103       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4104       PetscCall(ISDestroy(&perm));
4105     }
4106   }
4107   /* Handle */
4108   PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4109   PetscOptionsHeadEnd();
4110   PetscFunctionReturn(0);
4111 }
4112 
4113 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm, Vec *vec)
4114 {
4115   PetscFunctionBegin;
4116   PetscCall(DMCreateGlobalVector_Section_Private(dm, vec));
4117   /* PetscCall(VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM)); */
4118   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex));
4119   PetscCall(VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void))VecView_Plex_Native));
4120   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex));
4121   PetscCall(VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void))VecLoad_Plex_Native));
4122   PetscFunctionReturn(0);
4123 }
4124 
4125 static PetscErrorCode DMCreateLocalVector_Plex(DM dm, Vec *vec)
4126 {
4127   PetscFunctionBegin;
4128   PetscCall(DMCreateLocalVector_Section_Private(dm, vec));
4129   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex_Local));
4130   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex_Local));
4131   PetscFunctionReturn(0);
4132 }
4133 
4134 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
4135 {
4136   PetscInt depth, d;
4137 
4138   PetscFunctionBegin;
4139   PetscCall(DMPlexGetDepth(dm, &depth));
4140   if (depth == 1) {
4141     PetscCall(DMGetDimension(dm, &d));
4142     if (dim == 0) PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4143     else if (dim == d) PetscCall(DMPlexGetDepthStratum(dm, 1, pStart, pEnd));
4144     else {
4145       *pStart = 0;
4146       *pEnd   = 0;
4147     }
4148   } else {
4149     PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4150   }
4151   PetscFunctionReturn(0);
4152 }
4153 
4154 static PetscErrorCode DMGetNeighbors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
4155 {
4156   PetscSF            sf;
4157   PetscInt           niranks, njranks, n;
4158   const PetscMPIInt *iranks, *jranks;
4159   DM_Plex           *data = (DM_Plex *)dm->data;
4160 
4161   PetscFunctionBegin;
4162   PetscCall(DMGetPointSF(dm, &sf));
4163   if (!data->neighbors) {
4164     PetscCall(PetscSFSetUp(sf));
4165     PetscCall(PetscSFGetRootRanks(sf, &njranks, &jranks, NULL, NULL, NULL));
4166     PetscCall(PetscSFGetLeafRanks(sf, &niranks, &iranks, NULL, NULL));
4167     PetscCall(PetscMalloc1(njranks + niranks + 1, &data->neighbors));
4168     PetscCall(PetscArraycpy(data->neighbors + 1, jranks, njranks));
4169     PetscCall(PetscArraycpy(data->neighbors + njranks + 1, iranks, niranks));
4170     n = njranks + niranks;
4171     PetscCall(PetscSortRemoveDupsMPIInt(&n, data->neighbors + 1));
4172     /* The following cast should never fail: can't have more neighbors than PETSC_MPI_INT_MAX */
4173     PetscCall(PetscMPIIntCast(n, data->neighbors));
4174   }
4175   if (nranks) *nranks = data->neighbors[0];
4176   if (ranks) {
4177     if (data->neighbors[0]) *ranks = data->neighbors + 1;
4178     else *ranks = NULL;
4179   }
4180   PetscFunctionReturn(0);
4181 }
4182 
4183 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM, DM, Mat, Vec, Vec);
4184 
4185 static PetscErrorCode DMInitialize_Plex(DM dm)
4186 {
4187   PetscFunctionBegin;
4188   dm->ops->view                      = DMView_Plex;
4189   dm->ops->load                      = DMLoad_Plex;
4190   dm->ops->setfromoptions            = DMSetFromOptions_Plex;
4191   dm->ops->clone                     = DMClone_Plex;
4192   dm->ops->setup                     = DMSetUp_Plex;
4193   dm->ops->createlocalsection        = DMCreateLocalSection_Plex;
4194   dm->ops->createdefaultconstraints  = DMCreateDefaultConstraints_Plex;
4195   dm->ops->createglobalvector        = DMCreateGlobalVector_Plex;
4196   dm->ops->createlocalvector         = DMCreateLocalVector_Plex;
4197   dm->ops->getlocaltoglobalmapping   = NULL;
4198   dm->ops->createfieldis             = NULL;
4199   dm->ops->createcoordinatedm        = DMCreateCoordinateDM_Plex;
4200   dm->ops->createcoordinatefield     = DMCreateCoordinateField_Plex;
4201   dm->ops->getcoloring               = NULL;
4202   dm->ops->creatematrix              = DMCreateMatrix_Plex;
4203   dm->ops->createinterpolation       = DMCreateInterpolation_Plex;
4204   dm->ops->createmassmatrix          = DMCreateMassMatrix_Plex;
4205   dm->ops->createmassmatrixlumped    = DMCreateMassMatrixLumped_Plex;
4206   dm->ops->createinjection           = DMCreateInjection_Plex;
4207   dm->ops->refine                    = DMRefine_Plex;
4208   dm->ops->coarsen                   = DMCoarsen_Plex;
4209   dm->ops->refinehierarchy           = DMRefineHierarchy_Plex;
4210   dm->ops->coarsenhierarchy          = DMCoarsenHierarchy_Plex;
4211   dm->ops->extrude                   = DMExtrude_Plex;
4212   dm->ops->globaltolocalbegin        = NULL;
4213   dm->ops->globaltolocalend          = NULL;
4214   dm->ops->localtoglobalbegin        = NULL;
4215   dm->ops->localtoglobalend          = NULL;
4216   dm->ops->destroy                   = DMDestroy_Plex;
4217   dm->ops->createsubdm               = DMCreateSubDM_Plex;
4218   dm->ops->createsuperdm             = DMCreateSuperDM_Plex;
4219   dm->ops->getdimpoints              = DMGetDimPoints_Plex;
4220   dm->ops->locatepoints              = DMLocatePoints_Plex;
4221   dm->ops->projectfunctionlocal      = DMProjectFunctionLocal_Plex;
4222   dm->ops->projectfunctionlabellocal = DMProjectFunctionLabelLocal_Plex;
4223   dm->ops->projectfieldlocal         = DMProjectFieldLocal_Plex;
4224   dm->ops->projectfieldlabellocal    = DMProjectFieldLabelLocal_Plex;
4225   dm->ops->projectbdfieldlabellocal  = DMProjectBdFieldLabelLocal_Plex;
4226   dm->ops->computel2diff             = DMComputeL2Diff_Plex;
4227   dm->ops->computel2gradientdiff     = DMComputeL2GradientDiff_Plex;
4228   dm->ops->computel2fielddiff        = DMComputeL2FieldDiff_Plex;
4229   dm->ops->getneighbors              = DMGetNeighbors_Plex;
4230   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertBoundaryValues_C", DMPlexInsertBoundaryValues_Plex));
4231   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertTimeDerviativeBoundaryValues_C", DMPlexInsertTimeDerivativeBoundaryValues_Plex));
4232   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMSetUpGLVisViewer_C", DMSetUpGLVisViewer_Plex));
4233   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMCreateNeumannOverlap_C", DMCreateNeumannOverlap_Plex));
4234   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4235   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeGetDefault_C", DMPlexDistributeGetDefault_Plex));
4236   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeSetDefault_C", DMPlexDistributeSetDefault_Plex));
4237   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderGetDefault_C", DMPlexReorderGetDefault_Plex));
4238   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderSetDefault_C", DMPlexReorderSetDefault_Plex));
4239   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMInterpolateSolution_C", DMInterpolateSolution_Plex));
4240   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4241   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetOverlap_C", DMPlexSetOverlap_Plex));
4242   PetscFunctionReturn(0);
4243 }
4244 
4245 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
4246 {
4247   DM_Plex *mesh = (DM_Plex *)dm->data;
4248 
4249   PetscFunctionBegin;
4250   mesh->refct++;
4251   (*newdm)->data = mesh;
4252   PetscCall(PetscObjectChangeTypeName((PetscObject)*newdm, DMPLEX));
4253   PetscCall(DMInitialize_Plex(*newdm));
4254   PetscFunctionReturn(0);
4255 }
4256 
4257 /*MC
4258   DMPLEX = "plex" - A `DM` object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
4259                     In the local representation, Vecs contain all unknowns in the interior and shared boundary. This is
4260                     specified by a PetscSection object. Ownership in the global representation is determined by
4261                     ownership of the underlying `DMPLEX` points. This is specified by another `PetscSection` object.
4262 
4263   Options Database Keys:
4264 + -dm_refine_pre                     - Refine mesh before distribution
4265 + -dm_refine_uniform_pre             - Choose uniform or generator-based refinement
4266 + -dm_refine_volume_limit_pre        - Cell volume limit after pre-refinement using generator
4267 . -dm_distribute                     - Distribute mesh across processes
4268 . -dm_distribute_overlap             - Number of cells to overlap for distribution
4269 . -dm_refine                         - Refine mesh after distribution
4270 . -dm_plex_hash_location             - Use grid hashing for point location
4271 . -dm_plex_hash_box_faces <n,m,p>    - The number of divisions in each direction of the grid hash
4272 . -dm_plex_partition_balance         - Attempt to evenly divide points on partition boundary between processes
4273 . -dm_plex_remesh_bd                 - Allow changes to the boundary on remeshing
4274 . -dm_plex_max_projection_height     - Maximum mesh point height used to project locally
4275 . -dm_plex_regular_refinement        - Use special nested projection algorithm for regular refinement
4276 . -dm_plex_check_all                 - Perform all shecks below
4277 . -dm_plex_check_symmetry            - Check that the adjacency information in the mesh is symmetric
4278 . -dm_plex_check_skeleton <celltype> - Check that each cell has the correct number of vertices
4279 . -dm_plex_check_faces <celltype>    - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
4280 . -dm_plex_check_geometry            - Check that cells have positive volume
4281 . -dm_view :mesh.tex:ascii_latex     - View the mesh in LaTeX/TikZ
4282 . -dm_plex_view_scale <num>          - Scale the TikZ
4283 - -dm_plex_print_fem <num>           - View FEM assembly information, such as element vectors and matrices
4284 
4285   Level: intermediate
4286 
4287 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`, `PetscSection`
4288 M*/
4289 
4290 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
4291 {
4292   DM_Plex *mesh;
4293   PetscInt unit;
4294 
4295   PetscFunctionBegin;
4296   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4297   PetscCall(PetscNew(&mesh));
4298   dm->data = mesh;
4299 
4300   mesh->refct = 1;
4301   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection));
4302   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection));
4303   mesh->refinementUniform      = PETSC_TRUE;
4304   mesh->refinementLimit        = -1.0;
4305   mesh->distDefault            = PETSC_TRUE;
4306   mesh->reorderDefault         = DMPLEX_REORDER_DEFAULT_NOTSET;
4307   mesh->distributionName       = NULL;
4308   mesh->interpolated           = DMPLEX_INTERPOLATED_INVALID;
4309   mesh->interpolatedCollective = DMPLEX_INTERPOLATED_INVALID;
4310 
4311   PetscCall(PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner));
4312   mesh->remeshBd = PETSC_FALSE;
4313 
4314   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
4315 
4316   mesh->depthState    = -1;
4317   mesh->celltypeState = -1;
4318   mesh->printTol      = 1.0e-10;
4319 
4320   PetscCall(DMInitialize_Plex(dm));
4321   PetscFunctionReturn(0);
4322 }
4323 
4324 /*@
4325   DMPlexCreate - Creates a `DMPLEX` object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
4326 
4327   Collective
4328 
4329   Input Parameter:
4330 . comm - The communicator for the `DMPLEX` object
4331 
4332   Output Parameter:
4333 . mesh  - The `DMPLEX` object
4334 
4335   Level: beginner
4336 
4337 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`
4338 @*/
4339 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
4340 {
4341   PetscFunctionBegin;
4342   PetscValidPointer(mesh, 2);
4343   PetscCall(DMCreate(comm, mesh));
4344   PetscCall(DMSetType(*mesh, DMPLEX));
4345   PetscFunctionReturn(0);
4346 }
4347 
4348 /*@C
4349   DMPlexBuildFromCellListParallel - Build distributed `DMPLEX` topology from a list of vertices for each cell (common mesh generator output)
4350 
4351   Collective on dm
4352 
4353   Input Parameters:
4354 + dm - The `DM`
4355 . numCells - The number of cells owned by this process
4356 . numVertices - The number of vertices to be owned by this process, or `PETSC_DECIDE`
4357 . NVertices - The global number of vertices, or `PETSC_DETERMINE`
4358 . numCorners - The number of vertices for each cell
4359 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4360 
4361   Output Parameters:
4362 + vertexSF - (Optional) `PetscSF` describing complete vertex ownership
4363 - verticesAdjSaved - (Optional) vertex adjacency array
4364 
4365   Level: advanced
4366 
4367   Notes:
4368   Two triangles sharing a face
4369 .vb
4370 
4371         2
4372       / | \
4373      /  |  \
4374     /   |   \
4375    0  0 | 1  3
4376     \   |   /
4377      \  |  /
4378       \ | /
4379         1
4380 .ve
4381 would have input
4382 .vb
4383   numCells = 2, numVertices = 4
4384   cells = [0 1 2  1 3 2]
4385 .ve
4386 which would result in the `DMPLEX`
4387 .vb
4388 
4389         4
4390       / | \
4391      /  |  \
4392     /   |   \
4393    2  0 | 1  5
4394     \   |   /
4395      \  |  /
4396       \ | /
4397         3
4398 .ve
4399 
4400   Vertices are implicitly numbered consecutively 0,...,NVertices.
4401   Each rank owns a chunk of numVertices consecutive vertices.
4402   If numVertices is `PETSC_DECIDE`, PETSc will distribute them as evenly as possible using PetscLayout.
4403   If NVertices is `PETSC_DETERMINE` and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
4404   If only NVertices is `PETSC_DETERMINE`, it is computed as the sum of numVertices over all ranks.
4405 
4406   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
4407 
4408   Fortran Note:
4409   Not currently supported in Fortran.
4410 
4411 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildCoordinatesFromCellListParallel()`,
4412           `PetscSF`
4413 @*/
4414 PetscErrorCode DMPlexBuildFromCellListParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, const PetscInt cells[], PetscSF *vertexSF, PetscInt **verticesAdjSaved)
4415 {
4416   PetscSF     sfPoint;
4417   PetscLayout layout;
4418   PetscInt    numVerticesAdj, *verticesAdj, *cones, c, p;
4419 
4420   PetscFunctionBegin;
4421   PetscValidLogicalCollectiveInt(dm, NVertices, 4);
4422   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4423   /* Get/check global number of vertices */
4424   {
4425     PetscInt       NVerticesInCells, i;
4426     const PetscInt len = numCells * numCorners;
4427 
4428     /* NVerticesInCells = max(cells) + 1 */
4429     NVerticesInCells = PETSC_MIN_INT;
4430     for (i = 0; i < len; i++)
4431       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4432     ++NVerticesInCells;
4433     PetscCallMPI(MPI_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
4434 
4435     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
4436     else
4437       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
4438   }
4439   /* Count locally unique vertices */
4440   {
4441     PetscHSetI vhash;
4442     PetscInt   off = 0;
4443 
4444     PetscCall(PetscHSetICreate(&vhash));
4445     for (c = 0; c < numCells; ++c) {
4446       for (p = 0; p < numCorners; ++p) PetscCall(PetscHSetIAdd(vhash, cells[c * numCorners + p]));
4447     }
4448     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
4449     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
4450     else verticesAdj = *verticesAdjSaved;
4451     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
4452     PetscCall(PetscHSetIDestroy(&vhash));
4453     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
4454   }
4455   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
4456   /* Create cones */
4457   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
4458   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
4459   PetscCall(DMSetUp(dm));
4460   PetscCall(DMPlexGetCones(dm, &cones));
4461   for (c = 0; c < numCells; ++c) {
4462     for (p = 0; p < numCorners; ++p) {
4463       const PetscInt gv = cells[c * numCorners + p];
4464       PetscInt       lv;
4465 
4466       /* Positions within verticesAdj form 0-based local vertex numbering;
4467          we need to shift it by numCells to get correct DAG points (cells go first) */
4468       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
4469       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
4470       cones[c * numCorners + p] = lv + numCells;
4471     }
4472   }
4473   /* Build point sf */
4474   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
4475   PetscCall(PetscLayoutSetSize(layout, NVertices));
4476   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
4477   PetscCall(PetscLayoutSetBlockSize(layout, 1));
4478   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
4479   PetscCall(PetscLayoutDestroy(&layout));
4480   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
4481   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
4482   if (dm->sf) {
4483     const char *prefix;
4484 
4485     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
4486     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
4487   }
4488   PetscCall(DMSetPointSF(dm, sfPoint));
4489   PetscCall(PetscSFDestroy(&sfPoint));
4490   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)(*vertexSF), "Vertex Ownership SF"));
4491   /* Fill in the rest of the topology structure */
4492   PetscCall(DMPlexSymmetrize(dm));
4493   PetscCall(DMPlexStratify(dm));
4494   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4495   PetscFunctionReturn(0);
4496 }
4497 
4498 /*@C
4499   DMPlexBuildCoordinatesFromCellListParallel - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4500 
4501   Collective on dm
4502 
4503   Input Parameters:
4504 + dm - The `DM`
4505 . spaceDim - The spatial dimension used for coordinates
4506 . sfVert - `PetscSF` describing complete vertex ownership
4507 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4508 
4509   Level: advanced
4510 
4511   Fortran Note:
4512   Not currently supported in Fortran.
4513 
4514 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellListParallel()`
4515 @*/
4516 PetscErrorCode DMPlexBuildCoordinatesFromCellListParallel(DM dm, PetscInt spaceDim, PetscSF sfVert, const PetscReal vertexCoords[])
4517 {
4518   PetscSection coordSection;
4519   Vec          coordinates;
4520   PetscScalar *coords;
4521   PetscInt     numVertices, numVerticesAdj, coordSize, v, vStart, vEnd;
4522 
4523   PetscFunctionBegin;
4524   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4525   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
4526   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4527   PetscCall(DMSetCoordinateDim(dm, spaceDim));
4528   PetscCall(PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL));
4529   PetscCheck(vEnd - vStart == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Supplied sfVert has wrong number of leaves = %" PetscInt_FMT " != %" PetscInt_FMT " = vEnd - vStart", numVerticesAdj, vEnd - vStart);
4530   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4531   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4532   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
4533   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
4534   for (v = vStart; v < vEnd; ++v) {
4535     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
4536     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
4537   }
4538   PetscCall(PetscSectionSetUp(coordSection));
4539   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4540   PetscCall(VecCreate(PetscObjectComm((PetscObject)dm), &coordinates));
4541   PetscCall(VecSetBlockSize(coordinates, spaceDim));
4542   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4543   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4544   PetscCall(VecSetType(coordinates, VECSTANDARD));
4545   PetscCall(VecGetArray(coordinates, &coords));
4546   {
4547     MPI_Datatype coordtype;
4548 
4549     /* Need a temp buffer for coords if we have complex/single */
4550     PetscCallMPI(MPI_Type_contiguous(spaceDim, MPIU_SCALAR, &coordtype));
4551     PetscCallMPI(MPI_Type_commit(&coordtype));
4552 #if defined(PETSC_USE_COMPLEX)
4553     {
4554       PetscScalar *svertexCoords;
4555       PetscInt     i;
4556       PetscCall(PetscMalloc1(numVertices * spaceDim, &svertexCoords));
4557       for (i = 0; i < numVertices * spaceDim; i++) svertexCoords[i] = vertexCoords[i];
4558       PetscCall(PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4559       PetscCall(PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4560       PetscCall(PetscFree(svertexCoords));
4561     }
4562 #else
4563     PetscCall(PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4564     PetscCall(PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4565 #endif
4566     PetscCallMPI(MPI_Type_free(&coordtype));
4567   }
4568   PetscCall(VecRestoreArray(coordinates, &coords));
4569   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4570   PetscCall(VecDestroy(&coordinates));
4571   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4572   PetscFunctionReturn(0);
4573 }
4574 
4575 /*@
4576   DMPlexCreateFromCellListParallelPetsc - Create distributed `DMPLEX` from a list of vertices for each cell (common mesh generator output)
4577 
4578   Collective
4579 
4580   Input Parameters:
4581 + comm - The communicator
4582 . dim - The topological dimension of the mesh
4583 . numCells - The number of cells owned by this process
4584 . numVertices - The number of vertices owned by this process, or `PETSC_DECIDE`
4585 . NVertices - The global number of vertices, or `PETSC_DECIDE`
4586 . numCorners - The number of vertices for each cell
4587 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4588 . cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4589 . spaceDim - The spatial dimension used for coordinates
4590 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4591 
4592   Output Parameters:
4593 + dm - The `DM`
4594 . vertexSF - (Optional) `PetscSF` describing complete vertex ownership
4595 - verticesAdjSaved - (Optional) vertex adjacency array
4596 
4597   Level: intermediate
4598 
4599   Notes:
4600   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`,
4601   `DMPlexBuildFromCellListParallel()`, `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellListParallel()`
4602 
4603   See `DMPlexBuildFromCellListParallel()` for an example and details about the topology-related parameters.
4604 
4605   See `DMPlexBuildCoordinatesFromCellListParallel()` for details about the geometry-related parameters.
4606 
4607 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
4608 @*/
4609 PetscErrorCode DMPlexCreateFromCellListParallelPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, PetscInt **verticesAdj, DM *dm)
4610 {
4611   PetscSF sfVert;
4612 
4613   PetscFunctionBegin;
4614   PetscCall(DMCreate(comm, dm));
4615   PetscCall(DMSetType(*dm, DMPLEX));
4616   PetscValidLogicalCollectiveInt(*dm, dim, 2);
4617   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
4618   PetscCall(DMSetDimension(*dm, dim));
4619   PetscCall(DMPlexBuildFromCellListParallel(*dm, numCells, numVertices, NVertices, numCorners, cells, &sfVert, verticesAdj));
4620   if (interpolate) {
4621     DM idm;
4622 
4623     PetscCall(DMPlexInterpolate(*dm, &idm));
4624     PetscCall(DMDestroy(dm));
4625     *dm = idm;
4626   }
4627   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
4628   if (vertexSF) *vertexSF = sfVert;
4629   else PetscCall(PetscSFDestroy(&sfVert));
4630   PetscFunctionReturn(0);
4631 }
4632 
4633 /*@C
4634   DMPlexBuildFromCellList - Build `DMPLEX` topology from a list of vertices for each cell (common mesh generator output)
4635 
4636   Collective on dm
4637 
4638   Input Parameters:
4639 + dm - The `DM`
4640 . numCells - The number of cells owned by this process
4641 . numVertices - The number of vertices owned by this process, or `PETSC_DETERMINE`
4642 . numCorners - The number of vertices for each cell
4643 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4644 
4645   Level: advanced
4646 
4647   Notes:
4648   Two triangles sharing a face
4649 .vb
4650 
4651         2
4652       / | \
4653      /  |  \
4654     /   |   \
4655    0  0 | 1  3
4656     \   |   /
4657      \  |  /
4658       \ | /
4659         1
4660 .ve
4661 would have input
4662 .vb
4663   numCells = 2, numVertices = 4
4664   cells = [0 1 2  1 3 2]
4665 .ve
4666 which would result in the `DMPLEX`
4667 .vb
4668 
4669         4
4670       / | \
4671      /  |  \
4672     /   |   \
4673    2  0 | 1  5
4674     \   |   /
4675      \  |  /
4676       \ | /
4677         3
4678 .ve
4679 
4680   If numVertices is `PETSC_DETERMINE`, it is computed by PETSc as the maximum vertex index in cells + 1.
4681 
4682   Not currently supported in Fortran.
4683 
4684 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListPetsc()`
4685 @*/
4686 PetscErrorCode DMPlexBuildFromCellList(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const PetscInt cells[])
4687 {
4688   PetscInt *cones, c, p, dim;
4689 
4690   PetscFunctionBegin;
4691   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4692   PetscCall(DMGetDimension(dm, &dim));
4693   /* Get/check global number of vertices */
4694   {
4695     PetscInt       NVerticesInCells, i;
4696     const PetscInt len = numCells * numCorners;
4697 
4698     /* NVerticesInCells = max(cells) + 1 */
4699     NVerticesInCells = PETSC_MIN_INT;
4700     for (i = 0; i < len; i++)
4701       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4702     ++NVerticesInCells;
4703 
4704     if (numVertices == PETSC_DECIDE) numVertices = NVerticesInCells;
4705     else
4706       PetscCheck(numVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, numVertices, NVerticesInCells);
4707   }
4708   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
4709   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
4710   PetscCall(DMSetUp(dm));
4711   PetscCall(DMPlexGetCones(dm, &cones));
4712   for (c = 0; c < numCells; ++c) {
4713     for (p = 0; p < numCorners; ++p) cones[c * numCorners + p] = cells[c * numCorners + p] + numCells;
4714   }
4715   PetscCall(DMPlexSymmetrize(dm));
4716   PetscCall(DMPlexStratify(dm));
4717   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4718   PetscFunctionReturn(0);
4719 }
4720 
4721 /*@C
4722   DMPlexBuildCoordinatesFromCellList - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4723 
4724   Collective on dm
4725 
4726   Input Parameters:
4727 + dm - The `DM`
4728 . spaceDim - The spatial dimension used for coordinates
4729 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4730 
4731   Level: advanced
4732 
4733   Fortran Note:
4734   Not currently supported in Fortran.
4735 
4736 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellList()`
4737 @*/
4738 PetscErrorCode DMPlexBuildCoordinatesFromCellList(DM dm, PetscInt spaceDim, const PetscReal vertexCoords[])
4739 {
4740   PetscSection coordSection;
4741   Vec          coordinates;
4742   DM           cdm;
4743   PetscScalar *coords;
4744   PetscInt     v, vStart, vEnd, d;
4745 
4746   PetscFunctionBegin;
4747   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4748   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
4749   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4750   PetscCall(DMSetCoordinateDim(dm, spaceDim));
4751   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4752   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4753   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
4754   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
4755   for (v = vStart; v < vEnd; ++v) {
4756     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
4757     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
4758   }
4759   PetscCall(PetscSectionSetUp(coordSection));
4760 
4761   PetscCall(DMGetCoordinateDM(dm, &cdm));
4762   PetscCall(DMCreateLocalVector(cdm, &coordinates));
4763   PetscCall(VecSetBlockSize(coordinates, spaceDim));
4764   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4765   PetscCall(VecGetArrayWrite(coordinates, &coords));
4766   for (v = 0; v < vEnd - vStart; ++v) {
4767     for (d = 0; d < spaceDim; ++d) coords[v * spaceDim + d] = vertexCoords[v * spaceDim + d];
4768   }
4769   PetscCall(VecRestoreArrayWrite(coordinates, &coords));
4770   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4771   PetscCall(VecDestroy(&coordinates));
4772   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4773   PetscFunctionReturn(0);
4774 }
4775 
4776 /*@
4777   DMPlexCreateFromCellListPetsc - Create `DMPLEX` from a list of vertices for each cell (common mesh generator output), but only process 0 takes in the input
4778 
4779   Collective
4780 
4781   Input Parameters:
4782 + comm - The communicator
4783 . dim - The topological dimension of the mesh
4784 . numCells - The number of cells, only on process 0
4785 . numVertices - The number of vertices owned by this process, or `PETSC_DECIDE`, only on process 0
4786 . numCorners - The number of vertices for each cell, only on process 0
4787 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4788 . cells - An array of numCells*numCorners numbers, the vertices for each cell, only on process 0
4789 . spaceDim - The spatial dimension used for coordinates
4790 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex, only on process 0
4791 
4792   Output Parameter:
4793 . dm - The `DM`, which only has points on process 0
4794 
4795   Level: intermediate
4796 
4797   Notes:
4798   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`, `DMPlexBuildFromCellList()`,
4799   `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellList()`
4800 
4801   See `DMPlexBuildFromCellList()` for an example and details about the topology-related parameters.
4802   See `DMPlexBuildCoordinatesFromCellList()` for details about the geometry-related parameters.
4803   See `DMPlexCreateFromCellListParallelPetsc()` for parallel input
4804 
4805 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellList()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
4806 @*/
4807 PetscErrorCode DMPlexCreateFromCellListPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], DM *dm)
4808 {
4809   PetscMPIInt rank;
4810 
4811   PetscFunctionBegin;
4812   PetscCheck(dim, comm, PETSC_ERR_ARG_OUTOFRANGE, "This is not appropriate for 0-dimensional meshes. Consider either creating the DM using DMPlexCreateFromDAG(), by hand, or using DMSwarm.");
4813   PetscCallMPI(MPI_Comm_rank(comm, &rank));
4814   PetscCall(DMCreate(comm, dm));
4815   PetscCall(DMSetType(*dm, DMPLEX));
4816   PetscCall(DMSetDimension(*dm, dim));
4817   if (rank == 0) PetscCall(DMPlexBuildFromCellList(*dm, numCells, numVertices, numCorners, cells));
4818   else PetscCall(DMPlexBuildFromCellList(*dm, 0, 0, 0, NULL));
4819   if (interpolate) {
4820     DM idm;
4821 
4822     PetscCall(DMPlexInterpolate(*dm, &idm));
4823     PetscCall(DMDestroy(dm));
4824     *dm = idm;
4825   }
4826   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, vertexCoords));
4827   else PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, NULL));
4828   PetscFunctionReturn(0);
4829 }
4830 
4831 /*@
4832   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a DM
4833 
4834   Input Parameters:
4835 + dm - The empty DM object, usually from DMCreate() and DMSetDimension()
4836 . depth - The depth of the DAG
4837 . numPoints - Array of size depth + 1 containing the number of points at each depth
4838 . coneSize - The cone size of each point
4839 . cones - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
4840 . coneOrientations - The orientation of each cone point
4841 - vertexCoords - An array of numPoints[0]*spacedim numbers representing the coordinates of each vertex, with spacedim the value set via DMSetCoordinateDim()
4842 
4843   Output Parameter:
4844 . dm - The DM
4845 
4846   Note:
4847   Two triangles sharing a face would have input
4848 .vb
4849   depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
4850   cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
4851  vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
4852 .ve
4853 which would result in the DMPlex
4854 .vb
4855         4
4856       / | \
4857      /  |  \
4858     /   |   \
4859    2  0 | 1  5
4860     \   |   /
4861      \  |  /
4862       \ | /
4863         3
4864 .ve
4865  Notice that all points are numbered consecutively, unlike `DMPlexCreateFromCellListPetsc()`
4866 
4867   Level: advanced
4868 
4869 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
4870 @*/
4871 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
4872 {
4873   Vec          coordinates;
4874   PetscSection coordSection;
4875   PetscScalar *coords;
4876   PetscInt     coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
4877 
4878   PetscFunctionBegin;
4879   PetscCall(DMGetDimension(dm, &dim));
4880   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
4881   PetscCheck(dimEmbed >= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Embedding dimension %" PetscInt_FMT " cannot be less than intrinsic dimension %" PetscInt_FMT, dimEmbed, dim);
4882   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
4883   PetscCall(DMPlexSetChart(dm, pStart, pEnd));
4884   for (p = pStart; p < pEnd; ++p) {
4885     PetscCall(DMPlexSetConeSize(dm, p, coneSize[p - pStart]));
4886     if (firstVertex < 0 && !coneSize[p - pStart]) firstVertex = p - pStart;
4887   }
4888   PetscCheck(firstVertex >= 0 || !numPoints[0], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Expected %" PetscInt_FMT " vertices but could not find any", numPoints[0]);
4889   PetscCall(DMSetUp(dm)); /* Allocate space for cones */
4890   for (p = pStart, off = 0; p < pEnd; off += coneSize[p - pStart], ++p) {
4891     PetscCall(DMPlexSetCone(dm, p, &cones[off]));
4892     PetscCall(DMPlexSetConeOrientation(dm, p, &coneOrientations[off]));
4893   }
4894   PetscCall(DMPlexSymmetrize(dm));
4895   PetscCall(DMPlexStratify(dm));
4896   /* Build coordinates */
4897   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4898   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4899   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dimEmbed));
4900   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numPoints[0]));
4901   for (v = firstVertex; v < firstVertex + numPoints[0]; ++v) {
4902     PetscCall(PetscSectionSetDof(coordSection, v, dimEmbed));
4903     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed));
4904   }
4905   PetscCall(PetscSectionSetUp(coordSection));
4906   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4907   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
4908   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4909   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4910   PetscCall(VecSetBlockSize(coordinates, dimEmbed));
4911   PetscCall(VecSetType(coordinates, VECSTANDARD));
4912   if (vertexCoords) {
4913     PetscCall(VecGetArray(coordinates, &coords));
4914     for (v = 0; v < numPoints[0]; ++v) {
4915       PetscInt off;
4916 
4917       PetscCall(PetscSectionGetOffset(coordSection, v + firstVertex, &off));
4918       for (d = 0; d < dimEmbed; ++d) coords[off + d] = vertexCoords[v * dimEmbed + d];
4919     }
4920   }
4921   PetscCall(VecRestoreArray(coordinates, &coords));
4922   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4923   PetscCall(VecDestroy(&coordinates));
4924   PetscFunctionReturn(0);
4925 }
4926 
4927 /*@C
4928   DMPlexCreateCellVertexFromFile - Create a `DMPLEX` mesh from a simple cell-vertex file.
4929 
4930   Collective
4931 
4932 + comm        - The MPI communicator
4933 . filename    - Name of the .dat file
4934 - interpolate - Create faces and edges in the mesh
4935 
4936   Output Parameter:
4937 . dm  - The `DM` object representing the mesh
4938 
4939   Level: beginner
4940 
4941   Note:
4942   The format is the simplest possible:
4943 .vb
4944   Ne
4945   v0 v1 ... vk
4946   Nv
4947   x y z marker
4948 .ve
4949 
4950   Developer Note:
4951   Should use a `PetscViewer` not a filename
4952 
4953 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromFile()`, `DMPlexCreateMedFromFile()`, `DMPlexCreateGmsh()`, `DMPlexCreate()`
4954 @*/
4955 PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
4956 {
4957   DMLabel      marker;
4958   PetscViewer  viewer;
4959   Vec          coordinates;
4960   PetscSection coordSection;
4961   PetscScalar *coords;
4962   char         line[PETSC_MAX_PATH_LEN];
4963   PetscInt     dim = 3, cdim = 3, coordSize, v, c, d;
4964   PetscMPIInt  rank;
4965   int          snum, Nv, Nc, Ncn, Nl;
4966 
4967   PetscFunctionBegin;
4968   PetscCallMPI(MPI_Comm_rank(comm, &rank));
4969   PetscCall(PetscViewerCreate(comm, &viewer));
4970   PetscCall(PetscViewerSetType(viewer, PETSCVIEWERASCII));
4971   PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
4972   PetscCall(PetscViewerFileSetName(viewer, filename));
4973   if (rank == 0) {
4974     PetscCall(PetscViewerRead(viewer, line, 4, NULL, PETSC_STRING));
4975     snum = sscanf(line, "%d %d %d %d", &Nc, &Nv, &Ncn, &Nl);
4976     PetscCheck(snum == 4, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4977   } else {
4978     Nc = Nv = Ncn = Nl = 0;
4979   }
4980   PetscCall(DMCreate(comm, dm));
4981   PetscCall(DMSetType(*dm, DMPLEX));
4982   PetscCall(DMPlexSetChart(*dm, 0, Nc + Nv));
4983   PetscCall(DMSetDimension(*dm, dim));
4984   PetscCall(DMSetCoordinateDim(*dm, cdim));
4985   /* Read topology */
4986   if (rank == 0) {
4987     char     format[PETSC_MAX_PATH_LEN];
4988     PetscInt cone[8];
4989     int      vbuf[8], v;
4990 
4991     for (c = 0; c < Ncn; ++c) {
4992       format[c * 3 + 0] = '%';
4993       format[c * 3 + 1] = 'd';
4994       format[c * 3 + 2] = ' ';
4995     }
4996     format[Ncn * 3 - 1] = '\0';
4997     for (c = 0; c < Nc; ++c) PetscCall(DMPlexSetConeSize(*dm, c, Ncn));
4998     PetscCall(DMSetUp(*dm));
4999     for (c = 0; c < Nc; ++c) {
5000       PetscCall(PetscViewerRead(viewer, line, Ncn, NULL, PETSC_STRING));
5001       switch (Ncn) {
5002       case 2:
5003         snum = sscanf(line, format, &vbuf[0], &vbuf[1]);
5004         break;
5005       case 3:
5006         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2]);
5007         break;
5008       case 4:
5009         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3]);
5010         break;
5011       case 6:
5012         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5]);
5013         break;
5014       case 8:
5015         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);
5016         break;
5017       default:
5018         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No cell shape with %d vertices", Ncn);
5019       }
5020       PetscCheck(snum == Ncn, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5021       for (v = 0; v < Ncn; ++v) cone[v] = vbuf[v] + Nc;
5022       /* Hexahedra are inverted */
5023       if (Ncn == 8) {
5024         PetscInt tmp = cone[1];
5025         cone[1]      = cone[3];
5026         cone[3]      = tmp;
5027       }
5028       PetscCall(DMPlexSetCone(*dm, c, cone));
5029     }
5030   }
5031   PetscCall(DMPlexSymmetrize(*dm));
5032   PetscCall(DMPlexStratify(*dm));
5033   /* Read coordinates */
5034   PetscCall(DMGetCoordinateSection(*dm, &coordSection));
5035   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5036   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
5037   PetscCall(PetscSectionSetChart(coordSection, Nc, Nc + Nv));
5038   for (v = Nc; v < Nc + Nv; ++v) {
5039     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
5040     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
5041   }
5042   PetscCall(PetscSectionSetUp(coordSection));
5043   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
5044   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
5045   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5046   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
5047   PetscCall(VecSetBlockSize(coordinates, cdim));
5048   PetscCall(VecSetType(coordinates, VECSTANDARD));
5049   PetscCall(VecGetArray(coordinates, &coords));
5050   if (rank == 0) {
5051     char   format[PETSC_MAX_PATH_LEN];
5052     double x[3];
5053     int    l, val[3];
5054 
5055     if (Nl) {
5056       for (l = 0; l < Nl; ++l) {
5057         format[l * 3 + 0] = '%';
5058         format[l * 3 + 1] = 'd';
5059         format[l * 3 + 2] = ' ';
5060       }
5061       format[Nl * 3 - 1] = '\0';
5062       PetscCall(DMCreateLabel(*dm, "marker"));
5063       PetscCall(DMGetLabel(*dm, "marker", &marker));
5064     }
5065     for (v = 0; v < Nv; ++v) {
5066       PetscCall(PetscViewerRead(viewer, line, 3 + Nl, NULL, PETSC_STRING));
5067       snum = sscanf(line, "%lg %lg %lg", &x[0], &x[1], &x[2]);
5068       PetscCheck(snum == 3, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5069       switch (Nl) {
5070       case 0:
5071         snum = 0;
5072         break;
5073       case 1:
5074         snum = sscanf(line, format, &val[0]);
5075         break;
5076       case 2:
5077         snum = sscanf(line, format, &val[0], &val[1]);
5078         break;
5079       case 3:
5080         snum = sscanf(line, format, &val[0], &val[1], &val[2]);
5081         break;
5082       default:
5083         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Request support for %d labels", Nl);
5084       }
5085       PetscCheck(snum == Nl, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5086       for (d = 0; d < cdim; ++d) coords[v * cdim + d] = x[d];
5087       for (l = 0; l < Nl; ++l) PetscCall(DMLabelSetValue(marker, v + Nc, val[l]));
5088     }
5089   }
5090   PetscCall(VecRestoreArray(coordinates, &coords));
5091   PetscCall(DMSetCoordinatesLocal(*dm, coordinates));
5092   PetscCall(VecDestroy(&coordinates));
5093   PetscCall(PetscViewerDestroy(&viewer));
5094   if (interpolate) {
5095     DM      idm;
5096     DMLabel bdlabel;
5097 
5098     PetscCall(DMPlexInterpolate(*dm, &idm));
5099     PetscCall(DMDestroy(dm));
5100     *dm = idm;
5101 
5102     if (!Nl) {
5103       PetscCall(DMCreateLabel(*dm, "marker"));
5104       PetscCall(DMGetLabel(*dm, "marker", &bdlabel));
5105       PetscCall(DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel));
5106       PetscCall(DMPlexLabelComplete(*dm, bdlabel));
5107     }
5108   }
5109   PetscFunctionReturn(0);
5110 }
5111 
5112 /*@C
5113   DMPlexCreateFromFile - This takes a filename and produces a `DM`
5114 
5115   Collective
5116 
5117   Input Parameters:
5118 + comm - The communicator
5119 . filename - A file name
5120 . plexname - The object name of the resulting `DM`, also used for intra-datafile lookup by some formats
5121 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
5122 
5123   Output Parameter:
5124 . dm - The `DM`
5125 
5126   Options Database Key:
5127 . -dm_plex_create_from_hdf5_xdmf - use the `PETSC_VIEWER_HDF5_XDMF` format for reading HDF5
5128 
5129   Use -dm_plex_create_ prefix to pass options to the internal PetscViewer, e.g.
5130 $ -dm_plex_create_viewer_hdf5_collective
5131 
5132   Level: beginner
5133 
5134   Notes:
5135   Using `PETSCVIEWERHDF5` type with `PETSC_VIEWER_HDF5_PETSC` format, one can save multiple `DMPLEX`
5136   meshes in a single HDF5 file. This in turn requires one to name the `DMPLEX` object with `PetscObjectSetName()`
5137   before saving it with `DMView()` and before loading it with `DMLoad()` for identification of the mesh object.
5138   The input parameter name is thus used to name the `DMPLEX` object when `DMPlexCreateFromFile()` internally
5139   calls `DMLoad()`. Currently, name is ignored for other viewer types and/or formats.
5140 
5141 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`, `PetscObjectSetName()`, `DMView()`, `DMLoad()`
5142 @*/
5143 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], const char plexname[], PetscBool interpolate, DM *dm)
5144 {
5145   const char  extGmsh[]      = ".msh";
5146   const char  extGmsh2[]     = ".msh2";
5147   const char  extGmsh4[]     = ".msh4";
5148   const char  extCGNS[]      = ".cgns";
5149   const char  extExodus[]    = ".exo";
5150   const char  extExodus_e[]  = ".e";
5151   const char  extGenesis[]   = ".gen";
5152   const char  extFluent[]    = ".cas";
5153   const char  extHDF5[]      = ".h5";
5154   const char  extMed[]       = ".med";
5155   const char  extPLY[]       = ".ply";
5156   const char  extEGADSLite[] = ".egadslite";
5157   const char  extEGADS[]     = ".egads";
5158   const char  extIGES[]      = ".igs";
5159   const char  extSTEP[]      = ".stp";
5160   const char  extCV[]        = ".dat";
5161   size_t      len;
5162   PetscBool   isGmsh, isGmsh2, isGmsh4, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isMed, isPLY, isEGADSLite, isEGADS, isIGES, isSTEP, isCV;
5163   PetscMPIInt rank;
5164 
5165   PetscFunctionBegin;
5166   PetscValidCharPointer(filename, 2);
5167   if (plexname) PetscValidCharPointer(plexname, 3);
5168   PetscValidPointer(dm, 5);
5169   PetscCall(DMInitializePackage());
5170   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5171   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5172   PetscCall(PetscStrlen(filename, &len));
5173   PetscCheck(len, comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
5174 
5175 #define CheckExtension(extension__, is_extension__) \
5176   do { \
5177     PetscAssert(sizeof(extension__), comm, PETSC_ERR_PLIB, "Zero-size extension: %s", extension__); \
5178     /* don't count the null-terminator at the end */ \
5179     const size_t ext_len = sizeof(extension__) - 1; \
5180     if (len < ext_len) { \
5181       is_extension__ = PETSC_FALSE; \
5182     } else { \
5183       PetscCall(PetscStrncmp(filename + len - ext_len, extension__, ext_len, &is_extension__)); \
5184     } \
5185   } while (0)
5186 
5187   CheckExtension(extGmsh, isGmsh);
5188   CheckExtension(extGmsh2, isGmsh2);
5189   CheckExtension(extGmsh4, isGmsh4);
5190   CheckExtension(extCGNS, isCGNS);
5191   CheckExtension(extExodus, isExodus);
5192   if (!isExodus) CheckExtension(extExodus_e, isExodus);
5193   CheckExtension(extGenesis, isGenesis);
5194   CheckExtension(extFluent, isFluent);
5195   CheckExtension(extHDF5, isHDF5);
5196   CheckExtension(extMed, isMed);
5197   CheckExtension(extPLY, isPLY);
5198   CheckExtension(extEGADSLite, isEGADSLite);
5199   CheckExtension(extEGADS, isEGADS);
5200   CheckExtension(extIGES, isIGES);
5201   CheckExtension(extSTEP, isSTEP);
5202   CheckExtension(extCV, isCV);
5203 
5204 #undef CheckExtension
5205 
5206   if (isGmsh || isGmsh2 || isGmsh4) {
5207     PetscCall(DMPlexCreateGmshFromFile(comm, filename, interpolate, dm));
5208   } else if (isCGNS) {
5209     PetscCall(DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm));
5210   } else if (isExodus || isGenesis) {
5211     PetscCall(DMPlexCreateExodusFromFile(comm, filename, interpolate, dm));
5212   } else if (isFluent) {
5213     PetscCall(DMPlexCreateFluentFromFile(comm, filename, interpolate, dm));
5214   } else if (isHDF5) {
5215     PetscBool   load_hdf5_xdmf = PETSC_FALSE;
5216     PetscViewer viewer;
5217 
5218     /* PETSC_VIEWER_HDF5_XDMF is used if the filename ends with .xdmf.h5, or if -dm_plex_create_from_hdf5_xdmf option is present */
5219     PetscCall(PetscStrncmp(&filename[PetscMax(0, len - 8)], ".xdmf", 5, &load_hdf5_xdmf));
5220     PetscCall(PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &load_hdf5_xdmf, NULL));
5221     PetscCall(PetscViewerCreate(comm, &viewer));
5222     PetscCall(PetscViewerSetType(viewer, PETSCVIEWERHDF5));
5223     PetscCall(PetscViewerSetOptionsPrefix(viewer, "dm_plex_create_"));
5224     PetscCall(PetscViewerSetFromOptions(viewer));
5225     PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
5226     PetscCall(PetscViewerFileSetName(viewer, filename));
5227 
5228     PetscCall(DMCreate(comm, dm));
5229     PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5230     PetscCall(DMSetType(*dm, DMPLEX));
5231     if (load_hdf5_xdmf) PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF));
5232     PetscCall(DMLoad(*dm, viewer));
5233     if (load_hdf5_xdmf) PetscCall(PetscViewerPopFormat(viewer));
5234     PetscCall(PetscViewerDestroy(&viewer));
5235 
5236     if (interpolate) {
5237       DM idm;
5238 
5239       PetscCall(DMPlexInterpolate(*dm, &idm));
5240       PetscCall(DMDestroy(dm));
5241       *dm = idm;
5242     }
5243   } else if (isMed) {
5244     PetscCall(DMPlexCreateMedFromFile(comm, filename, interpolate, dm));
5245   } else if (isPLY) {
5246     PetscCall(DMPlexCreatePLYFromFile(comm, filename, interpolate, dm));
5247   } else if (isEGADSLite || isEGADS || isIGES || isSTEP) {
5248     if (isEGADSLite) PetscCall(DMPlexCreateEGADSLiteFromFile(comm, filename, dm));
5249     else PetscCall(DMPlexCreateEGADSFromFile(comm, filename, dm));
5250     if (!interpolate) {
5251       DM udm;
5252 
5253       PetscCall(DMPlexUninterpolate(*dm, &udm));
5254       PetscCall(DMDestroy(dm));
5255       *dm = udm;
5256     }
5257   } else if (isCV) {
5258     PetscCall(DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm));
5259   } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
5260   PetscCall(PetscStrlen(plexname, &len));
5261   if (len) PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5262   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5263   PetscFunctionReturn(0);
5264 }
5265