xref: /petsc/src/dm/impls/plex/plexcreate.c (revision d5b43468fb8780a8feea140ccd6fa3e6a50411cc)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h> /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>   /*I   "petscdmplex.h"   I*/
4 #include <petscsf.h>
5 #include <petscdmplextransform.h>
6 #include <petsc/private/kernels/blockmatmult.h>
7 #include <petsc/private/kernels/blockinvert.h>
8 
9 PetscLogEvent DMPLEX_CreateFromFile, DMPLEX_BuildFromCellList, DMPLEX_BuildCoordinatesFromCellList;
10 
11 /* External function declarations here */
12 static PetscErrorCode DMInitialize_Plex(DM dm);
13 
14 /* This copies internal things in the Plex structure that we generally want when making a new, related Plex */
15 PetscErrorCode DMPlexCopy_Internal(DM dmin, PetscBool copyPeriodicity, PetscBool copyOverlap, DM dmout)
16 {
17   const PetscReal         *maxCell, *Lstart, *L;
18   PetscBool                dist;
19   DMPlexReorderDefaultFlag reorder;
20 
21   PetscFunctionBegin;
22   if (copyPeriodicity) {
23     PetscCall(DMGetPeriodicity(dmin, &maxCell, &Lstart, &L));
24     PetscCall(DMSetPeriodicity(dmout, maxCell, Lstart, L));
25   }
26   PetscCall(DMPlexDistributeGetDefault(dmin, &dist));
27   PetscCall(DMPlexDistributeSetDefault(dmout, dist));
28   PetscCall(DMPlexReorderGetDefault(dmin, &reorder));
29   PetscCall(DMPlexReorderSetDefault(dmout, reorder));
30   ((DM_Plex *)dmout->data)->useHashLocation = ((DM_Plex *)dmin->data)->useHashLocation;
31   if (copyOverlap) PetscCall(DMPlexSetOverlap_Plex(dmout, dmin, 0));
32   PetscFunctionReturn(0);
33 }
34 
35 /* Replace dm with the contents of ndm, and then destroy ndm
36    - Share the DM_Plex structure
37    - Share the coordinates
38    - Share the SF
39 */
40 PetscErrorCode DMPlexReplace_Internal(DM dm, DM *ndm)
41 {
42   PetscSF          sf;
43   DM               dmNew = *ndm, coordDM, coarseDM;
44   Vec              coords;
45   const PetscReal *maxCell, *Lstart, *L;
46   PetscInt         dim, cdim;
47 
48   PetscFunctionBegin;
49   if (dm == dmNew) {
50     PetscCall(DMDestroy(ndm));
51     PetscFunctionReturn(0);
52   }
53   dm->setupcalled = dmNew->setupcalled;
54   PetscCall(DMGetDimension(dmNew, &dim));
55   PetscCall(DMSetDimension(dm, dim));
56   PetscCall(DMGetCoordinateDim(dmNew, &cdim));
57   PetscCall(DMSetCoordinateDim(dm, cdim));
58   PetscCall(DMGetPointSF(dmNew, &sf));
59   PetscCall(DMSetPointSF(dm, sf));
60   PetscCall(DMGetCoordinateDM(dmNew, &coordDM));
61   PetscCall(DMGetCoordinatesLocal(dmNew, &coords));
62   PetscCall(DMSetCoordinateDM(dm, coordDM));
63   PetscCall(DMSetCoordinatesLocal(dm, coords));
64   PetscCall(DMGetCellCoordinateDM(dmNew, &coordDM));
65   PetscCall(DMGetCellCoordinatesLocal(dmNew, &coords));
66   PetscCall(DMSetCellCoordinateDM(dm, coordDM));
67   PetscCall(DMSetCellCoordinatesLocal(dm, coords));
68   /* Do not want to create the coordinate field if it does not already exist, so do not call DMGetCoordinateField() */
69   PetscCall(DMFieldDestroy(&dm->coordinates[0].field));
70   dm->coordinates[0].field            = dmNew->coordinates[0].field;
71   ((DM_Plex *)dmNew->data)->coordFunc = ((DM_Plex *)dm->data)->coordFunc;
72   PetscCall(DMGetPeriodicity(dmNew, &maxCell, &Lstart, &L));
73   PetscCall(DMSetPeriodicity(dm, maxCell, Lstart, L));
74   PetscCall(DMDestroy_Plex(dm));
75   PetscCall(DMInitialize_Plex(dm));
76   dm->data = dmNew->data;
77   ((DM_Plex *)dmNew->data)->refct++;
78   PetscCall(DMDestroyLabelLinkList_Internal(dm));
79   PetscCall(DMCopyLabels(dmNew, dm, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL));
80   PetscCall(DMGetCoarseDM(dmNew, &coarseDM));
81   PetscCall(DMSetCoarseDM(dm, coarseDM));
82   PetscCall(DMDestroy(ndm));
83   PetscFunctionReturn(0);
84 }
85 
86 /* Swap dm with the contents of dmNew
87    - Swap the DM_Plex structure
88    - Swap the coordinates
89    - Swap the point PetscSF
90 */
91 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
92 {
93   DM          coordDMA, coordDMB;
94   Vec         coordsA, coordsB;
95   PetscSF     sfA, sfB;
96   DMField     fieldTmp;
97   void       *tmp;
98   DMLabelLink listTmp;
99   DMLabel     depthTmp;
100   PetscInt    tmpI;
101 
102   PetscFunctionBegin;
103   if (dmA == dmB) PetscFunctionReturn(0);
104   PetscCall(DMGetPointSF(dmA, &sfA));
105   PetscCall(DMGetPointSF(dmB, &sfB));
106   PetscCall(PetscObjectReference((PetscObject)sfA));
107   PetscCall(DMSetPointSF(dmA, sfB));
108   PetscCall(DMSetPointSF(dmB, sfA));
109   PetscCall(PetscObjectDereference((PetscObject)sfA));
110 
111   PetscCall(DMGetCoordinateDM(dmA, &coordDMA));
112   PetscCall(DMGetCoordinateDM(dmB, &coordDMB));
113   PetscCall(PetscObjectReference((PetscObject)coordDMA));
114   PetscCall(DMSetCoordinateDM(dmA, coordDMB));
115   PetscCall(DMSetCoordinateDM(dmB, coordDMA));
116   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
117 
118   PetscCall(DMGetCoordinatesLocal(dmA, &coordsA));
119   PetscCall(DMGetCoordinatesLocal(dmB, &coordsB));
120   PetscCall(PetscObjectReference((PetscObject)coordsA));
121   PetscCall(DMSetCoordinatesLocal(dmA, coordsB));
122   PetscCall(DMSetCoordinatesLocal(dmB, coordsA));
123   PetscCall(PetscObjectDereference((PetscObject)coordsA));
124 
125   PetscCall(DMGetCellCoordinateDM(dmA, &coordDMA));
126   PetscCall(DMGetCellCoordinateDM(dmB, &coordDMB));
127   PetscCall(PetscObjectReference((PetscObject)coordDMA));
128   PetscCall(DMSetCellCoordinateDM(dmA, coordDMB));
129   PetscCall(DMSetCellCoordinateDM(dmB, coordDMA));
130   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
131 
132   PetscCall(DMGetCellCoordinatesLocal(dmA, &coordsA));
133   PetscCall(DMGetCellCoordinatesLocal(dmB, &coordsB));
134   PetscCall(PetscObjectReference((PetscObject)coordsA));
135   PetscCall(DMSetCellCoordinatesLocal(dmA, coordsB));
136   PetscCall(DMSetCellCoordinatesLocal(dmB, coordsA));
137   PetscCall(PetscObjectDereference((PetscObject)coordsA));
138 
139   fieldTmp                  = dmA->coordinates[0].field;
140   dmA->coordinates[0].field = dmB->coordinates[0].field;
141   dmB->coordinates[0].field = fieldTmp;
142   fieldTmp                  = dmA->coordinates[1].field;
143   dmA->coordinates[1].field = dmB->coordinates[1].field;
144   dmB->coordinates[1].field = fieldTmp;
145   tmp                       = dmA->data;
146   dmA->data                 = dmB->data;
147   dmB->data                 = tmp;
148   listTmp                   = dmA->labels;
149   dmA->labels               = dmB->labels;
150   dmB->labels               = listTmp;
151   depthTmp                  = dmA->depthLabel;
152   dmA->depthLabel           = dmB->depthLabel;
153   dmB->depthLabel           = depthTmp;
154   depthTmp                  = dmA->celltypeLabel;
155   dmA->celltypeLabel        = dmB->celltypeLabel;
156   dmB->celltypeLabel        = depthTmp;
157   tmpI                      = dmA->levelup;
158   dmA->levelup              = dmB->levelup;
159   dmB->levelup              = tmpI;
160   PetscFunctionReturn(0);
161 }
162 
163 static PetscErrorCode DMPlexInterpolateInPlace_Internal(DM dm)
164 {
165   DM idm;
166 
167   PetscFunctionBegin;
168   PetscCall(DMPlexInterpolate(dm, &idm));
169   PetscCall(DMPlexCopyCoordinates(dm, idm));
170   PetscCall(DMPlexReplace_Internal(dm, &idm));
171   PetscFunctionReturn(0);
172 }
173 
174 /*@C
175   DMPlexCreateCoordinateSpace - Creates a finite element space for the coordinates
176 
177   Collective
178 
179   Input Parameters:
180 + DM        - The DM
181 . degree    - The degree of the finite element or PETSC_DECIDE
182 - coordFunc - An optional function to map new points from refinement to the surface
183 
184   Level: advanced
185 
186 .seealso: `PetscFECreateLagrange()`, `DMGetCoordinateDM()`
187 @*/
188 PetscErrorCode DMPlexCreateCoordinateSpace(DM dm, PetscInt degree, PetscPointFunc coordFunc)
189 {
190   DM_Plex     *mesh = (DM_Plex *)dm->data;
191   DM           cdm;
192   PetscDS      cds;
193   PetscFE      fe;
194   PetscClassId id;
195 
196   PetscFunctionBegin;
197   PetscCall(DMGetCoordinateDM(dm, &cdm));
198   PetscCall(DMGetDS(cdm, &cds));
199   PetscCall(PetscDSGetDiscretization(cds, 0, (PetscObject *)&fe));
200   PetscCall(PetscObjectGetClassId((PetscObject)fe, &id));
201   if (id != PETSCFE_CLASSID) {
202     PetscBool simplex;
203     PetscInt  dim, dE, qorder;
204 
205     PetscCall(DMGetDimension(dm, &dim));
206     PetscCall(DMGetCoordinateDim(dm, &dE));
207     qorder = degree;
208     PetscObjectOptionsBegin((PetscObject)cdm);
209     PetscCall(PetscOptionsBoundedInt("-coord_dm_default_quadrature_order", "Quadrature order is one less than quadrature points per edge", "DMPlexCreateCoordinateSpace", qorder, &qorder, NULL, 0));
210     PetscOptionsEnd();
211     if (degree == PETSC_DECIDE) fe = NULL;
212     else {
213       PetscCall(DMPlexIsSimplex(dm, &simplex));
214       PetscCall(PetscFECreateLagrange(PETSC_COMM_SELF, dim, dE, simplex, degree, qorder, &fe));
215     }
216     PetscCall(DMProjectCoordinates(dm, fe));
217     PetscCall(PetscFEDestroy(&fe));
218   }
219   mesh->coordFunc = coordFunc;
220   PetscFunctionReturn(0);
221 }
222 
223 /*@
224   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
225 
226   Collective
227 
228   Input Parameters:
229 + comm - The communicator for the DM object
230 . dim - The spatial dimension
231 . simplex - Flag for simplicial cells, otherwise they are tensor product cells
232 . interpolate - Flag to create intermediate mesh pieces (edges, faces)
233 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
234 
235   Output Parameter:
236 . dm - The DM object
237 
238   Level: beginner
239 
240 .seealso: `DMSetType()`, `DMCreate()`
241 @*/
242 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscReal refinementLimit, DM *newdm)
243 {
244   DM          dm;
245   PetscMPIInt rank;
246 
247   PetscFunctionBegin;
248   PetscCall(DMCreate(comm, &dm));
249   PetscCall(DMSetType(dm, DMPLEX));
250   PetscCall(DMSetDimension(dm, dim));
251   PetscCallMPI(MPI_Comm_rank(comm, &rank));
252   switch (dim) {
253   case 2:
254     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "triangular"));
255     else PetscCall(PetscObjectSetName((PetscObject)dm, "quadrilateral"));
256     break;
257   case 3:
258     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "tetrahedral"));
259     else PetscCall(PetscObjectSetName((PetscObject)dm, "hexahedral"));
260     break;
261   default:
262     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
263   }
264   if (rank) {
265     PetscInt numPoints[2] = {0, 0};
266     PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL));
267   } else {
268     switch (dim) {
269     case 2:
270       if (simplex) {
271         PetscInt    numPoints[2]        = {4, 2};
272         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
273         PetscInt    cones[6]            = {2, 3, 4, 5, 4, 3};
274         PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
275         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
276 
277         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
278       } else {
279         PetscInt    numPoints[2]        = {6, 2};
280         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
281         PetscInt    cones[8]            = {2, 3, 4, 5, 3, 6, 7, 4};
282         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
283         PetscScalar vertexCoords[12]    = {-1.0, -0.5, 0.0, -0.5, 0.0, 0.5, -1.0, 0.5, 1.0, -0.5, 1.0, 0.5};
284 
285         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
286       }
287       break;
288     case 3:
289       if (simplex) {
290         PetscInt    numPoints[2]        = {5, 2};
291         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
292         PetscInt    cones[8]            = {4, 3, 5, 2, 5, 3, 4, 6};
293         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
294         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0};
295 
296         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
297       } else {
298         PetscInt    numPoints[2]         = {12, 2};
299         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
300         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9, 5, 4, 10, 11, 7, 12, 13, 8};
301         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
302         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5, -1.0, 0.5, -0.5, 0.0, 0.5, -0.5, 0.0, -0.5, -0.5, -1.0, -0.5, 0.5, 0.0, -0.5, 0.5, 0.0, 0.5, 0.5, -1.0, 0.5, 0.5, 1.0, 0.5, -0.5, 1.0, -0.5, -0.5, 1.0, -0.5, 0.5, 1.0, 0.5, 0.5};
303 
304         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
305       }
306       break;
307     default:
308       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
309     }
310   }
311   *newdm = dm;
312   if (refinementLimit > 0.0) {
313     DM          rdm;
314     const char *name;
315 
316     PetscCall(DMPlexSetRefinementUniform(*newdm, PETSC_FALSE));
317     PetscCall(DMPlexSetRefinementLimit(*newdm, refinementLimit));
318     PetscCall(DMRefine(*newdm, comm, &rdm));
319     PetscCall(PetscObjectGetName((PetscObject)*newdm, &name));
320     PetscCall(PetscObjectSetName((PetscObject)rdm, name));
321     PetscCall(DMDestroy(newdm));
322     *newdm = rdm;
323   }
324   if (interpolate) {
325     DM idm;
326 
327     PetscCall(DMPlexInterpolate(*newdm, &idm));
328     PetscCall(DMDestroy(newdm));
329     *newdm = idm;
330   }
331   PetscFunctionReturn(0);
332 }
333 
334 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
335 {
336   const PetscInt numVertices    = 2;
337   PetscInt       markerRight    = 1;
338   PetscInt       markerLeft     = 1;
339   PetscBool      markerSeparate = PETSC_FALSE;
340   Vec            coordinates;
341   PetscSection   coordSection;
342   PetscScalar   *coords;
343   PetscInt       coordSize;
344   PetscMPIInt    rank;
345   PetscInt       cdim = 1, v;
346 
347   PetscFunctionBegin;
348   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
349   if (markerSeparate) {
350     markerRight = 2;
351     markerLeft  = 1;
352   }
353   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
354   if (rank == 0) {
355     PetscCall(DMPlexSetChart(dm, 0, numVertices));
356     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
357     PetscCall(DMSetLabelValue(dm, "marker", 0, markerLeft));
358     PetscCall(DMSetLabelValue(dm, "marker", 1, markerRight));
359   }
360   PetscCall(DMPlexSymmetrize(dm));
361   PetscCall(DMPlexStratify(dm));
362   /* Build coordinates */
363   PetscCall(DMSetCoordinateDim(dm, cdim));
364   PetscCall(DMGetCoordinateSection(dm, &coordSection));
365   PetscCall(PetscSectionSetNumFields(coordSection, 1));
366   PetscCall(PetscSectionSetChart(coordSection, 0, numVertices));
367   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
368   for (v = 0; v < numVertices; ++v) {
369     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
370     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
371   }
372   PetscCall(PetscSectionSetUp(coordSection));
373   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
374   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
375   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
376   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
377   PetscCall(VecSetBlockSize(coordinates, cdim));
378   PetscCall(VecSetType(coordinates, VECSTANDARD));
379   PetscCall(VecGetArray(coordinates, &coords));
380   coords[0] = lower[0];
381   coords[1] = upper[0];
382   PetscCall(VecRestoreArray(coordinates, &coords));
383   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
384   PetscCall(VecDestroy(&coordinates));
385   PetscFunctionReturn(0);
386 }
387 
388 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
389 {
390   const PetscInt numVertices    = (edges[0] + 1) * (edges[1] + 1);
391   const PetscInt numEdges       = edges[0] * (edges[1] + 1) + (edges[0] + 1) * edges[1];
392   PetscInt       markerTop      = 1;
393   PetscInt       markerBottom   = 1;
394   PetscInt       markerRight    = 1;
395   PetscInt       markerLeft     = 1;
396   PetscBool      markerSeparate = PETSC_FALSE;
397   Vec            coordinates;
398   PetscSection   coordSection;
399   PetscScalar   *coords;
400   PetscInt       coordSize;
401   PetscMPIInt    rank;
402   PetscInt       v, vx, vy;
403 
404   PetscFunctionBegin;
405   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
406   if (markerSeparate) {
407     markerTop    = 3;
408     markerBottom = 1;
409     markerRight  = 2;
410     markerLeft   = 4;
411   }
412   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
413   if (rank == 0) {
414     PetscInt e, ex, ey;
415 
416     PetscCall(DMPlexSetChart(dm, 0, numEdges + numVertices));
417     for (e = 0; e < numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
418     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
419     for (vx = 0; vx <= edges[0]; vx++) {
420       for (ey = 0; ey < edges[1]; ey++) {
421         PetscInt edge   = vx * edges[1] + ey + edges[0] * (edges[1] + 1);
422         PetscInt vertex = ey * (edges[0] + 1) + vx + numEdges;
423         PetscInt cone[2];
424 
425         cone[0] = vertex;
426         cone[1] = vertex + edges[0] + 1;
427         PetscCall(DMPlexSetCone(dm, edge, cone));
428         if (vx == edges[0]) {
429           PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
430           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
431           if (ey == edges[1] - 1) {
432             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
433             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerRight));
434           }
435         } else if (vx == 0) {
436           PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
437           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
438           if (ey == edges[1] - 1) {
439             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
440             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft));
441           }
442         }
443       }
444     }
445     for (vy = 0; vy <= edges[1]; vy++) {
446       for (ex = 0; ex < edges[0]; ex++) {
447         PetscInt edge   = vy * edges[0] + ex;
448         PetscInt vertex = vy * (edges[0] + 1) + ex + numEdges;
449         PetscInt cone[2];
450 
451         cone[0] = vertex;
452         cone[1] = vertex + 1;
453         PetscCall(DMPlexSetCone(dm, edge, cone));
454         if (vy == edges[1]) {
455           PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
456           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
457           if (ex == edges[0] - 1) {
458             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
459             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerTop));
460           }
461         } else if (vy == 0) {
462           PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
463           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
464           if (ex == edges[0] - 1) {
465             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
466             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom));
467           }
468         }
469       }
470     }
471   }
472   PetscCall(DMPlexSymmetrize(dm));
473   PetscCall(DMPlexStratify(dm));
474   /* Build coordinates */
475   PetscCall(DMSetCoordinateDim(dm, 2));
476   PetscCall(DMGetCoordinateSection(dm, &coordSection));
477   PetscCall(PetscSectionSetNumFields(coordSection, 1));
478   PetscCall(PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices));
479   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 2));
480   for (v = numEdges; v < numEdges + numVertices; ++v) {
481     PetscCall(PetscSectionSetDof(coordSection, v, 2));
482     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 2));
483   }
484   PetscCall(PetscSectionSetUp(coordSection));
485   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
486   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
487   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
488   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
489   PetscCall(VecSetBlockSize(coordinates, 2));
490   PetscCall(VecSetType(coordinates, VECSTANDARD));
491   PetscCall(VecGetArray(coordinates, &coords));
492   for (vy = 0; vy <= edges[1]; ++vy) {
493     for (vx = 0; vx <= edges[0]; ++vx) {
494       coords[(vy * (edges[0] + 1) + vx) * 2 + 0] = lower[0] + ((upper[0] - lower[0]) / edges[0]) * vx;
495       coords[(vy * (edges[0] + 1) + vx) * 2 + 1] = lower[1] + ((upper[1] - lower[1]) / edges[1]) * vy;
496     }
497   }
498   PetscCall(VecRestoreArray(coordinates, &coords));
499   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
500   PetscCall(VecDestroy(&coordinates));
501   PetscFunctionReturn(0);
502 }
503 
504 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
505 {
506   PetscInt     vertices[3], numVertices;
507   PetscInt     numFaces       = 2 * faces[0] * faces[1] + 2 * faces[1] * faces[2] + 2 * faces[0] * faces[2];
508   PetscInt     markerTop      = 1;
509   PetscInt     markerBottom   = 1;
510   PetscInt     markerFront    = 1;
511   PetscInt     markerBack     = 1;
512   PetscInt     markerRight    = 1;
513   PetscInt     markerLeft     = 1;
514   PetscBool    markerSeparate = PETSC_FALSE;
515   Vec          coordinates;
516   PetscSection coordSection;
517   PetscScalar *coords;
518   PetscInt     coordSize;
519   PetscMPIInt  rank;
520   PetscInt     v, vx, vy, vz;
521   PetscInt     voffset, iface = 0, cone[4];
522 
523   PetscFunctionBegin;
524   PetscCheck(faces[0] >= 1 && faces[1] >= 1 && faces[2] >= 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
525   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
526   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
527   if (markerSeparate) {
528     markerBottom = 1;
529     markerTop    = 2;
530     markerFront  = 3;
531     markerBack   = 4;
532     markerRight  = 5;
533     markerLeft   = 6;
534   }
535   vertices[0] = faces[0] + 1;
536   vertices[1] = faces[1] + 1;
537   vertices[2] = faces[2] + 1;
538   numVertices = vertices[0] * vertices[1] * vertices[2];
539   if (rank == 0) {
540     PetscInt f;
541 
542     PetscCall(DMPlexSetChart(dm, 0, numFaces + numVertices));
543     for (f = 0; f < numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
544     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
545 
546     /* Side 0 (Top) */
547     for (vy = 0; vy < faces[1]; vy++) {
548       for (vx = 0; vx < faces[0]; vx++) {
549         voffset = numFaces + vertices[0] * vertices[1] * (vertices[2] - 1) + vy * vertices[0] + vx;
550         cone[0] = voffset;
551         cone[1] = voffset + 1;
552         cone[2] = voffset + vertices[0] + 1;
553         cone[3] = voffset + vertices[0];
554         PetscCall(DMPlexSetCone(dm, iface, cone));
555         PetscCall(DMSetLabelValue(dm, "marker", iface, markerTop));
556         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerTop));
557         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerTop));
558         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerTop));
559         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerTop));
560         iface++;
561       }
562     }
563 
564     /* Side 1 (Bottom) */
565     for (vy = 0; vy < faces[1]; vy++) {
566       for (vx = 0; vx < faces[0]; vx++) {
567         voffset = numFaces + vy * (faces[0] + 1) + vx;
568         cone[0] = voffset + 1;
569         cone[1] = voffset;
570         cone[2] = voffset + vertices[0];
571         cone[3] = voffset + vertices[0] + 1;
572         PetscCall(DMPlexSetCone(dm, iface, cone));
573         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBottom));
574         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBottom));
575         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBottom));
576         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerBottom));
577         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerBottom));
578         iface++;
579       }
580     }
581 
582     /* Side 2 (Front) */
583     for (vz = 0; vz < faces[2]; vz++) {
584       for (vx = 0; vx < faces[0]; vx++) {
585         voffset = numFaces + vz * vertices[0] * vertices[1] + vx;
586         cone[0] = voffset;
587         cone[1] = voffset + 1;
588         cone[2] = voffset + vertices[0] * vertices[1] + 1;
589         cone[3] = voffset + vertices[0] * vertices[1];
590         PetscCall(DMPlexSetCone(dm, iface, cone));
591         PetscCall(DMSetLabelValue(dm, "marker", iface, markerFront));
592         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerFront));
593         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerFront));
594         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerFront));
595         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerFront));
596         iface++;
597       }
598     }
599 
600     /* Side 3 (Back) */
601     for (vz = 0; vz < faces[2]; vz++) {
602       for (vx = 0; vx < faces[0]; vx++) {
603         voffset = numFaces + vz * vertices[0] * vertices[1] + vertices[0] * (vertices[1] - 1) + vx;
604         cone[0] = voffset + vertices[0] * vertices[1];
605         cone[1] = voffset + vertices[0] * vertices[1] + 1;
606         cone[2] = voffset + 1;
607         cone[3] = voffset;
608         PetscCall(DMPlexSetCone(dm, iface, cone));
609         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBack));
610         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBack));
611         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBack));
612         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerBack));
613         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerBack));
614         iface++;
615       }
616     }
617 
618     /* Side 4 (Left) */
619     for (vz = 0; vz < faces[2]; vz++) {
620       for (vy = 0; vy < faces[1]; vy++) {
621         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0];
622         cone[0] = voffset;
623         cone[1] = voffset + vertices[0] * vertices[1];
624         cone[2] = voffset + vertices[0] * vertices[1] + vertices[0];
625         cone[3] = voffset + vertices[0];
626         PetscCall(DMPlexSetCone(dm, iface, cone));
627         PetscCall(DMSetLabelValue(dm, "marker", iface, markerLeft));
628         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerLeft));
629         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerLeft));
630         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[1] + 0, markerLeft));
631         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerLeft));
632         iface++;
633       }
634     }
635 
636     /* Side 5 (Right) */
637     for (vz = 0; vz < faces[2]; vz++) {
638       for (vy = 0; vy < faces[1]; vy++) {
639         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0] + faces[0];
640         cone[0] = voffset + vertices[0] * vertices[1];
641         cone[1] = voffset;
642         cone[2] = voffset + vertices[0];
643         cone[3] = voffset + vertices[0] * vertices[1] + vertices[0];
644         PetscCall(DMPlexSetCone(dm, iface, cone));
645         PetscCall(DMSetLabelValue(dm, "marker", iface, markerRight));
646         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerRight));
647         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerRight));
648         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerRight));
649         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerRight));
650         iface++;
651       }
652     }
653   }
654   PetscCall(DMPlexSymmetrize(dm));
655   PetscCall(DMPlexStratify(dm));
656   /* Build coordinates */
657   PetscCall(DMSetCoordinateDim(dm, 3));
658   PetscCall(DMGetCoordinateSection(dm, &coordSection));
659   PetscCall(PetscSectionSetNumFields(coordSection, 1));
660   PetscCall(PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices));
661   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 3));
662   for (v = numFaces; v < numFaces + numVertices; ++v) {
663     PetscCall(PetscSectionSetDof(coordSection, v, 3));
664     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 3));
665   }
666   PetscCall(PetscSectionSetUp(coordSection));
667   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
668   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
669   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
670   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
671   PetscCall(VecSetBlockSize(coordinates, 3));
672   PetscCall(VecSetType(coordinates, VECSTANDARD));
673   PetscCall(VecGetArray(coordinates, &coords));
674   for (vz = 0; vz <= faces[2]; ++vz) {
675     for (vy = 0; vy <= faces[1]; ++vy) {
676       for (vx = 0; vx <= faces[0]; ++vx) {
677         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 0] = lower[0] + ((upper[0] - lower[0]) / faces[0]) * vx;
678         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 1] = lower[1] + ((upper[1] - lower[1]) / faces[1]) * vy;
679         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 2] = lower[2] + ((upper[2] - lower[2]) / faces[2]) * vz;
680       }
681     }
682   }
683   PetscCall(VecRestoreArray(coordinates, &coords));
684   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
685   PetscCall(VecDestroy(&coordinates));
686   PetscFunctionReturn(0);
687 }
688 
689 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate)
690 {
691   PetscFunctionBegin;
692   PetscValidLogicalCollectiveInt(dm, dim, 2);
693   PetscCall(DMSetDimension(dm, dim - 1));
694   PetscCall(DMSetCoordinateDim(dm, dim));
695   switch (dim) {
696   case 1:
697     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(dm, lower, upper, faces));
698     break;
699   case 2:
700     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(dm, lower, upper, faces));
701     break;
702   case 3:
703     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(dm, lower, upper, faces));
704     break;
705   default:
706     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Dimension not supported: %" PetscInt_FMT, dim);
707   }
708   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
709   PetscFunctionReturn(0);
710 }
711 
712 /*@C
713   DMPlexCreateBoxSurfaceMesh - Creates a mesh on the surface of the tensor product of unit intervals (box) using tensor cells (hexahedra).
714 
715   Collective
716 
717   Input Parameters:
718 + comm        - The communicator for the DM object
719 . dim         - The spatial dimension of the box, so the resulting mesh is has dimension dim-1
720 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
721 . lower       - The lower left corner, or NULL for (0, 0, 0)
722 . upper       - The upper right corner, or NULL for (1, 1, 1)
723 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
724 
725   Output Parameter:
726 . dm  - The DM object
727 
728   Level: beginner
729 
730 .seealso: `DMSetFromOptions()`, `DMPlexCreateBoxMesh()`, `DMPlexCreateFromFile()`, `DMSetType()`, `DMCreate()`
731 @*/
732 PetscErrorCode DMPlexCreateBoxSurfaceMesh(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate, DM *dm)
733 {
734   PetscInt  fac[3] = {1, 1, 1};
735   PetscReal low[3] = {0, 0, 0};
736   PetscReal upp[3] = {1, 1, 1};
737 
738   PetscFunctionBegin;
739   PetscCall(DMCreate(comm, dm));
740   PetscCall(DMSetType(*dm, DMPLEX));
741   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(*dm, dim, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, interpolate));
742   PetscFunctionReturn(0);
743 }
744 
745 static PetscErrorCode DMPlexCreateLineMesh_Internal(DM dm, PetscInt segments, PetscReal lower, PetscReal upper, DMBoundaryType bd)
746 {
747   PetscInt     i, fStart, fEnd, numCells = 0, numVerts = 0;
748   PetscInt     numPoints[2], *coneSize, *cones, *coneOrientations;
749   PetscScalar *vertexCoords;
750   PetscReal    L, maxCell;
751   PetscBool    markerSeparate = PETSC_FALSE;
752   PetscInt     markerLeft = 1, faceMarkerLeft = 1;
753   PetscInt     markerRight = 1, faceMarkerRight = 2;
754   PetscBool    wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
755   PetscMPIInt  rank;
756 
757   PetscFunctionBegin;
758   PetscValidPointer(dm, 1);
759 
760   PetscCall(DMSetDimension(dm, 1));
761   PetscCall(DMCreateLabel(dm, "marker"));
762   PetscCall(DMCreateLabel(dm, "Face Sets"));
763 
764   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
765   if (rank == 0) numCells = segments;
766   if (rank == 0) numVerts = segments + (wrap ? 0 : 1);
767 
768   numPoints[0] = numVerts;
769   numPoints[1] = numCells;
770   PetscCall(PetscMalloc4(numCells + numVerts, &coneSize, numCells * 2, &cones, numCells + numVerts, &coneOrientations, numVerts, &vertexCoords));
771   PetscCall(PetscArrayzero(coneOrientations, numCells + numVerts));
772   for (i = 0; i < numCells; ++i) coneSize[i] = 2;
773   for (i = 0; i < numVerts; ++i) coneSize[numCells + i] = 0;
774   for (i = 0; i < numCells; ++i) {
775     cones[2 * i]     = numCells + i % numVerts;
776     cones[2 * i + 1] = numCells + (i + 1) % numVerts;
777   }
778   for (i = 0; i < numVerts; ++i) vertexCoords[i] = lower + (upper - lower) * ((PetscReal)i / (PetscReal)numCells);
779   PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
780   PetscCall(PetscFree4(coneSize, cones, coneOrientations, vertexCoords));
781 
782   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
783   if (markerSeparate) {
784     markerLeft  = faceMarkerLeft;
785     markerRight = faceMarkerRight;
786   }
787   if (!wrap && rank == 0) {
788     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
789     PetscCall(DMSetLabelValue(dm, "marker", fStart, markerLeft));
790     PetscCall(DMSetLabelValue(dm, "marker", fEnd - 1, markerRight));
791     PetscCall(DMSetLabelValue(dm, "Face Sets", fStart, faceMarkerLeft));
792     PetscCall(DMSetLabelValue(dm, "Face Sets", fEnd - 1, faceMarkerRight));
793   }
794   if (wrap) {
795     L       = upper - lower;
796     maxCell = (PetscReal)1.1 * (L / (PetscReal)PetscMax(1, segments));
797     PetscCall(DMSetPeriodicity(dm, &maxCell, &lower, &L));
798   }
799   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
800   PetscFunctionReturn(0);
801 }
802 
803 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
804 {
805   DM      boundary, vol;
806   DMLabel bdlabel;
807 
808   PetscFunctionBegin;
809   PetscValidPointer(dm, 1);
810   for (PetscInt i = 0; i < dim; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
811   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &boundary));
812   PetscCall(DMSetType(boundary, DMPLEX));
813   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(boundary, dim, faces, lower, upper, PETSC_FALSE));
814   PetscCall(DMPlexGenerate(boundary, NULL, interpolate, &vol));
815   PetscCall(DMGetLabel(vol, "marker", &bdlabel));
816   if (bdlabel) PetscCall(DMPlexLabelComplete(vol, bdlabel));
817   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_FALSE, vol));
818   PetscCall(DMPlexReplace_Internal(dm, &vol));
819   PetscCall(DMDestroy(&boundary));
820   PetscFunctionReturn(0);
821 }
822 
823 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
824 {
825   DMLabel     cutLabel  = NULL;
826   PetscInt    markerTop = 1, faceMarkerTop = 1;
827   PetscInt    markerBottom = 1, faceMarkerBottom = 1;
828   PetscInt    markerFront = 1, faceMarkerFront = 1;
829   PetscInt    markerBack = 1, faceMarkerBack = 1;
830   PetscInt    markerRight = 1, faceMarkerRight = 1;
831   PetscInt    markerLeft = 1, faceMarkerLeft = 1;
832   PetscInt    dim;
833   PetscBool   markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
834   PetscMPIInt rank;
835 
836   PetscFunctionBegin;
837   PetscCall(DMGetDimension(dm, &dim));
838   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
839   PetscCall(DMCreateLabel(dm, "marker"));
840   PetscCall(DMCreateLabel(dm, "Face Sets"));
841   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
842   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST || bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST || bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
843     if (cutMarker) {
844       PetscCall(DMCreateLabel(dm, "periodic_cut"));
845       PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
846     }
847   }
848   switch (dim) {
849   case 2:
850     faceMarkerTop    = 3;
851     faceMarkerBottom = 1;
852     faceMarkerRight  = 2;
853     faceMarkerLeft   = 4;
854     break;
855   case 3:
856     faceMarkerBottom = 1;
857     faceMarkerTop    = 2;
858     faceMarkerFront  = 3;
859     faceMarkerBack   = 4;
860     faceMarkerRight  = 5;
861     faceMarkerLeft   = 6;
862     break;
863   default:
864     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
865   }
866   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
867   if (markerSeparate) {
868     markerBottom = faceMarkerBottom;
869     markerTop    = faceMarkerTop;
870     markerFront  = faceMarkerFront;
871     markerBack   = faceMarkerBack;
872     markerRight  = faceMarkerRight;
873     markerLeft   = faceMarkerLeft;
874   }
875   {
876     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
877     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
878     const PetscInt numZEdges    = rank == 0 ? edges[2] : 0;
879     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0] + 1) : 0;
880     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1] + 1) : 0;
881     const PetscInt numZVertices = rank == 0 ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2] + 1) : 0;
882     const PetscInt numCells     = numXEdges * numYEdges * numZEdges;
883     const PetscInt numXFaces    = numYEdges * numZEdges;
884     const PetscInt numYFaces    = numXEdges * numZEdges;
885     const PetscInt numZFaces    = numXEdges * numYEdges;
886     const PetscInt numTotXFaces = numXVertices * numXFaces;
887     const PetscInt numTotYFaces = numYVertices * numYFaces;
888     const PetscInt numTotZFaces = numZVertices * numZFaces;
889     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
890     const PetscInt numTotXEdges = numXEdges * numYVertices * numZVertices;
891     const PetscInt numTotYEdges = numYEdges * numXVertices * numZVertices;
892     const PetscInt numTotZEdges = numZEdges * numXVertices * numYVertices;
893     const PetscInt numVertices  = numXVertices * numYVertices * numZVertices;
894     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
895     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
896     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
897     const PetscInt firstYFace   = firstXFace + numTotXFaces;
898     const PetscInt firstZFace   = firstYFace + numTotYFaces;
899     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
900     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
901     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
902     Vec            coordinates;
903     PetscSection   coordSection;
904     PetscScalar   *coords;
905     PetscInt       coordSize;
906     PetscInt       v, vx, vy, vz;
907     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
908 
909     PetscCall(DMPlexSetChart(dm, 0, numCells + numFaces + numEdges + numVertices));
910     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
911     for (f = firstXFace; f < firstXFace + numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
912     for (e = firstXEdge; e < firstXEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
913     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
914     /* Build cells */
915     for (fz = 0; fz < numZEdges; ++fz) {
916       for (fy = 0; fy < numYEdges; ++fy) {
917         for (fx = 0; fx < numXEdges; ++fx) {
918           PetscInt cell  = (fz * numYEdges + fy) * numXEdges + fx;
919           PetscInt faceB = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
920           PetscInt faceT = firstZFace + (fy * numXEdges + fx) * numZVertices + ((fz + 1) % numZVertices);
921           PetscInt faceF = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
922           PetscInt faceK = firstYFace + (fz * numXEdges + fx) * numYVertices + ((fy + 1) % numYVertices);
923           PetscInt faceL = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
924           PetscInt faceR = firstXFace + (fz * numYEdges + fy) * numXVertices + ((fx + 1) % numXVertices);
925           /* B,  T,  F,  K,  R,  L */
926           PetscInt ornt[6] = {-2, 0, 0, -3, 0, -2}; /* ??? */
927           PetscInt cone[6];
928 
929           /* no boundary twisting in 3D */
930           cone[0] = faceB;
931           cone[1] = faceT;
932           cone[2] = faceF;
933           cone[3] = faceK;
934           cone[4] = faceR;
935           cone[5] = faceL;
936           PetscCall(DMPlexSetCone(dm, cell, cone));
937           PetscCall(DMPlexSetConeOrientation(dm, cell, ornt));
938           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
939           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
940           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
941         }
942       }
943     }
944     /* Build x faces */
945     for (fz = 0; fz < numZEdges; ++fz) {
946       for (fy = 0; fy < numYEdges; ++fy) {
947         for (fx = 0; fx < numXVertices; ++fx) {
948           PetscInt face    = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
949           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
950           PetscInt edgeR   = firstZEdge + (((fy + 1) % numYVertices) * numXVertices + fx) * numZEdges + fz;
951           PetscInt edgeB   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
952           PetscInt edgeT   = firstYEdge + (((fz + 1) % numZVertices) * numXVertices + fx) * numYEdges + fy;
953           PetscInt ornt[4] = {0, 0, -1, -1};
954           PetscInt cone[4];
955 
956           if (dim == 3) {
957             /* markers */
958             if (bdX != DM_BOUNDARY_PERIODIC) {
959               if (fx == numXVertices - 1) {
960                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight));
961                 PetscCall(DMSetLabelValue(dm, "marker", face, markerRight));
962               } else if (fx == 0) {
963                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft));
964                 PetscCall(DMSetLabelValue(dm, "marker", face, markerLeft));
965               }
966             }
967           }
968           cone[0] = edgeB;
969           cone[1] = edgeR;
970           cone[2] = edgeT;
971           cone[3] = edgeL;
972           PetscCall(DMPlexSetCone(dm, face, cone));
973           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
974         }
975       }
976     }
977     /* Build y faces */
978     for (fz = 0; fz < numZEdges; ++fz) {
979       for (fx = 0; fx < numXEdges; ++fx) {
980         for (fy = 0; fy < numYVertices; ++fy) {
981           PetscInt face    = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
982           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
983           PetscInt edgeR   = firstZEdge + (fy * numXVertices + ((fx + 1) % numXVertices)) * numZEdges + fz;
984           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
985           PetscInt edgeT   = firstXEdge + (((fz + 1) % numZVertices) * numYVertices + fy) * numXEdges + fx;
986           PetscInt ornt[4] = {0, 0, -1, -1};
987           PetscInt cone[4];
988 
989           if (dim == 3) {
990             /* markers */
991             if (bdY != DM_BOUNDARY_PERIODIC) {
992               if (fy == numYVertices - 1) {
993                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack));
994                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBack));
995               } else if (fy == 0) {
996                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront));
997                 PetscCall(DMSetLabelValue(dm, "marker", face, markerFront));
998               }
999             }
1000           }
1001           cone[0] = edgeB;
1002           cone[1] = edgeR;
1003           cone[2] = edgeT;
1004           cone[3] = edgeL;
1005           PetscCall(DMPlexSetCone(dm, face, cone));
1006           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1007         }
1008       }
1009     }
1010     /* Build z faces */
1011     for (fy = 0; fy < numYEdges; ++fy) {
1012       for (fx = 0; fx < numXEdges; ++fx) {
1013         for (fz = 0; fz < numZVertices; fz++) {
1014           PetscInt face    = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
1015           PetscInt edgeL   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
1016           PetscInt edgeR   = firstYEdge + (fz * numXVertices + ((fx + 1) % numXVertices)) * numYEdges + fy;
1017           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1018           PetscInt edgeT   = firstXEdge + (fz * numYVertices + ((fy + 1) % numYVertices)) * numXEdges + fx;
1019           PetscInt ornt[4] = {0, 0, -1, -1};
1020           PetscInt cone[4];
1021 
1022           if (dim == 2) {
1023             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges - 1) {
1024               edgeR += numYEdges - 1 - 2 * fy;
1025               ornt[1] = -1;
1026             }
1027             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges - 1) {
1028               edgeT += numXEdges - 1 - 2 * fx;
1029               ornt[2] = 0;
1030             }
1031             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1032             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1033           } else {
1034             /* markers */
1035             if (bdZ != DM_BOUNDARY_PERIODIC) {
1036               if (fz == numZVertices - 1) {
1037                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop));
1038                 PetscCall(DMSetLabelValue(dm, "marker", face, markerTop));
1039               } else if (fz == 0) {
1040                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom));
1041                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBottom));
1042               }
1043             }
1044           }
1045           cone[0] = edgeB;
1046           cone[1] = edgeR;
1047           cone[2] = edgeT;
1048           cone[3] = edgeL;
1049           PetscCall(DMPlexSetCone(dm, face, cone));
1050           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1051         }
1052       }
1053     }
1054     /* Build Z edges*/
1055     for (vy = 0; vy < numYVertices; vy++) {
1056       for (vx = 0; vx < numXVertices; vx++) {
1057         for (ez = 0; ez < numZEdges; ez++) {
1058           const PetscInt edge    = firstZEdge + (vy * numXVertices + vx) * numZEdges + ez;
1059           const PetscInt vertexB = firstVertex + (ez * numYVertices + vy) * numXVertices + vx;
1060           const PetscInt vertexT = firstVertex + (((ez + 1) % numZVertices) * numYVertices + vy) * numXVertices + vx;
1061           PetscInt       cone[2];
1062 
1063           cone[0] = vertexB;
1064           cone[1] = vertexT;
1065           PetscCall(DMPlexSetCone(dm, edge, cone));
1066           if (dim == 3) {
1067             if (bdX != DM_BOUNDARY_PERIODIC) {
1068               if (vx == numXVertices - 1) {
1069                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1070                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1071                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1072               } else if (vx == 0) {
1073                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1074                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1075                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1076               }
1077             }
1078             if (bdY != DM_BOUNDARY_PERIODIC) {
1079               if (vy == numYVertices - 1) {
1080                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1081                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1082                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1083               } else if (vy == 0) {
1084                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1085                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1086                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1087               }
1088             }
1089           }
1090         }
1091       }
1092     }
1093     /* Build Y edges*/
1094     for (vz = 0; vz < numZVertices; vz++) {
1095       for (vx = 0; vx < numXVertices; vx++) {
1096         for (ey = 0; ey < numYEdges; ey++) {
1097           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges - 1) ? (numXVertices - vx - 1) : (vz * numYVertices + ((ey + 1) % numYVertices)) * numXVertices + vx;
1098           const PetscInt edge    = firstYEdge + (vz * numXVertices + vx) * numYEdges + ey;
1099           const PetscInt vertexF = firstVertex + (vz * numYVertices + ey) * numXVertices + vx;
1100           const PetscInt vertexK = firstVertex + nextv;
1101           PetscInt       cone[2];
1102 
1103           cone[0] = vertexF;
1104           cone[1] = vertexK;
1105           PetscCall(DMPlexSetCone(dm, edge, cone));
1106           if (dim == 2) {
1107             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1108               if (vx == numXVertices - 1) {
1109                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight));
1110                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1111                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1112                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1113               } else if (vx == 0) {
1114                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft));
1115                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1116                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1117                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1118               }
1119             } else {
1120               if (vx == 0 && cutLabel) {
1121                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1122                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1123                 if (ey == numYEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1124               }
1125             }
1126           } else {
1127             if (bdX != DM_BOUNDARY_PERIODIC) {
1128               if (vx == numXVertices - 1) {
1129                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1130                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1131                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1132               } else if (vx == 0) {
1133                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1134                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1135                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1136               }
1137             }
1138             if (bdZ != DM_BOUNDARY_PERIODIC) {
1139               if (vz == numZVertices - 1) {
1140                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1141                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1142                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1143               } else if (vz == 0) {
1144                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1145                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1146                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1147               }
1148             }
1149           }
1150         }
1151       }
1152     }
1153     /* Build X edges*/
1154     for (vz = 0; vz < numZVertices; vz++) {
1155       for (vy = 0; vy < numYVertices; vy++) {
1156         for (ex = 0; ex < numXEdges; ex++) {
1157           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges - 1) ? (numYVertices - vy - 1) * numXVertices : (vz * numYVertices + vy) * numXVertices + (ex + 1) % numXVertices;
1158           const PetscInt edge    = firstXEdge + (vz * numYVertices + vy) * numXEdges + ex;
1159           const PetscInt vertexL = firstVertex + (vz * numYVertices + vy) * numXVertices + ex;
1160           const PetscInt vertexR = firstVertex + nextv;
1161           PetscInt       cone[2];
1162 
1163           cone[0] = vertexL;
1164           cone[1] = vertexR;
1165           PetscCall(DMPlexSetCone(dm, edge, cone));
1166           if (dim == 2) {
1167             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1168               if (vy == numYVertices - 1) {
1169                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop));
1170                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1171                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1172                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1173               } else if (vy == 0) {
1174                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom));
1175                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1176                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1177                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1178               }
1179             } else {
1180               if (vy == 0 && cutLabel) {
1181                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1182                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1183                 if (ex == numXEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1184               }
1185             }
1186           } else {
1187             if (bdY != DM_BOUNDARY_PERIODIC) {
1188               if (vy == numYVertices - 1) {
1189                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1190                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1191                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1192               } else if (vy == 0) {
1193                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1194                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1195                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1196               }
1197             }
1198             if (bdZ != DM_BOUNDARY_PERIODIC) {
1199               if (vz == numZVertices - 1) {
1200                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1201                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1202                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1203               } else if (vz == 0) {
1204                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1205                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1206                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1207               }
1208             }
1209           }
1210         }
1211       }
1212     }
1213     PetscCall(DMPlexSymmetrize(dm));
1214     PetscCall(DMPlexStratify(dm));
1215     /* Build coordinates */
1216     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1217     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1218     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1219     PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVertices));
1220     for (v = firstVertex; v < firstVertex + numVertices; ++v) {
1221       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1222       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1223     }
1224     PetscCall(PetscSectionSetUp(coordSection));
1225     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1226     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1227     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1228     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1229     PetscCall(VecSetBlockSize(coordinates, dim));
1230     PetscCall(VecSetType(coordinates, VECSTANDARD));
1231     PetscCall(VecGetArray(coordinates, &coords));
1232     for (vz = 0; vz < numZVertices; ++vz) {
1233       for (vy = 0; vy < numYVertices; ++vy) {
1234         for (vx = 0; vx < numXVertices; ++vx) {
1235           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * vx;
1236           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * vy;
1237           if (dim == 3) coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 2] = lower[2] + ((upper[2] - lower[2]) / numZEdges) * vz;
1238         }
1239       }
1240     }
1241     PetscCall(VecRestoreArray(coordinates, &coords));
1242     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1243     PetscCall(VecDestroy(&coordinates));
1244   }
1245   PetscFunctionReturn(0);
1246 }
1247 
1248 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1249 {
1250   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1251   PetscInt       fac[3] = {0, 0, 0}, d;
1252 
1253   PetscFunctionBegin;
1254   PetscValidPointer(dm, 1);
1255   PetscValidLogicalCollectiveInt(dm, dim, 2);
1256   PetscCall(DMSetDimension(dm, dim));
1257   for (d = 0; d < dim; ++d) {
1258     fac[d] = faces[d];
1259     bdt[d] = periodicity[d];
1260   }
1261   PetscCall(DMPlexCreateCubeMesh_Internal(dm, lower, upper, fac, bdt[0], bdt[1], bdt[2]));
1262   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST || periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST || (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
1263     PetscReal L[3]       = {-1., -1., 0.};
1264     PetscReal maxCell[3] = {-1., -1., 0.};
1265 
1266     for (d = 0; d < dim; ++d) {
1267       if (periodicity[d] != DM_BOUNDARY_NONE) {
1268         L[d]       = upper[d] - lower[d];
1269         maxCell[d] = 1.1 * (L[d] / PetscMax(1, faces[d]));
1270       }
1271     }
1272     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1273   }
1274   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
1275   PetscFunctionReturn(0);
1276 }
1277 
1278 static PetscErrorCode DMPlexCreateBoxMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1279 {
1280   PetscFunctionBegin;
1281   if (dim == 1) PetscCall(DMPlexCreateLineMesh_Internal(dm, faces[0], lower[0], upper[0], periodicity[0]));
1282   else if (simplex) PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1283   else PetscCall(DMPlexCreateBoxMesh_Tensor_Internal(dm, dim, faces, lower, upper, periodicity));
1284   if (!interpolate && dim > 1 && !simplex) {
1285     DM udm;
1286 
1287     PetscCall(DMPlexUninterpolate(dm, &udm));
1288     PetscCall(DMPlexCopyCoordinates(dm, udm));
1289     PetscCall(DMPlexReplace_Internal(dm, &udm));
1290   }
1291   PetscFunctionReturn(0);
1292 }
1293 
1294 /*@C
1295   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
1296 
1297   Collective
1298 
1299   Input Parameters:
1300 + comm        - The communicator for the DM object
1301 . dim         - The spatial dimension
1302 . simplex     - PETSC_TRUE for simplices, PETSC_FALSE for tensor cells
1303 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1304 . lower       - The lower left corner, or NULL for (0, 0, 0)
1305 . upper       - The upper right corner, or NULL for (1, 1, 1)
1306 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
1307 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1308 
1309   Output Parameter:
1310 . dm  - The DM object
1311 
1312   Note: If you want to customize this mesh using options, you just need to
1313 $  DMCreate(comm, &dm);
1314 $  DMSetType(dm, DMPLEX);
1315 $  DMSetFromOptions(dm);
1316 and use the options on the DMSetFromOptions() page.
1317 
1318   Here is the numbering returned for 2 faces in each direction for tensor cells:
1319 $ 10---17---11---18----12
1320 $  |         |         |
1321 $  |         |         |
1322 $ 20    2   22    3    24
1323 $  |         |         |
1324 $  |         |         |
1325 $  7---15----8---16----9
1326 $  |         |         |
1327 $  |         |         |
1328 $ 19    0   21    1   23
1329 $  |         |         |
1330 $  |         |         |
1331 $  4---13----5---14----6
1332 
1333 and for simplicial cells
1334 
1335 $ 14----8---15----9----16
1336 $  |\     5  |\      7 |
1337 $  | \       | \       |
1338 $ 13   2    14    3    15
1339 $  | 4   \   | 6   \   |
1340 $  |       \ |       \ |
1341 $ 11----6---12----7----13
1342 $  |\        |\        |
1343 $  | \    1  | \     3 |
1344 $ 10   0    11    1    12
1345 $  | 0   \   | 2   \   |
1346 $  |       \ |       \ |
1347 $  8----4----9----5----10
1348 
1349   Level: beginner
1350 
1351 .seealso: `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
1352 @*/
1353 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
1354 {
1355   PetscInt       fac[3] = {1, 1, 1};
1356   PetscReal      low[3] = {0, 0, 0};
1357   PetscReal      upp[3] = {1, 1, 1};
1358   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1359 
1360   PetscFunctionBegin;
1361   PetscCall(DMCreate(comm, dm));
1362   PetscCall(DMSetType(*dm, DMPLEX));
1363   PetscCall(DMPlexCreateBoxMesh_Internal(*dm, dim, simplex, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt, interpolate));
1364   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1365   PetscFunctionReturn(0);
1366 }
1367 
1368 static PetscErrorCode DMPlexCreateWedgeBoxMesh_Internal(DM dm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1369 {
1370   DM       bdm, vol;
1371   PetscInt i;
1372 
1373   PetscFunctionBegin;
1374   for (i = 0; i < 3; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity not yet supported");
1375   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &bdm));
1376   PetscCall(DMSetType(bdm, DMPLEX));
1377   PetscCall(DMSetDimension(bdm, 2));
1378   PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(bdm, 2, faces, lower, upper, periodicity, PETSC_TRUE));
1379   PetscCall(DMPlexExtrude(bdm, faces[2], upper[2] - lower[2], PETSC_TRUE, PETSC_FALSE, NULL, NULL, &vol));
1380   PetscCall(DMDestroy(&bdm));
1381   PetscCall(DMPlexReplace_Internal(dm, &vol));
1382   if (lower[2] != 0.0) {
1383     Vec          v;
1384     PetscScalar *x;
1385     PetscInt     cDim, n;
1386 
1387     PetscCall(DMGetCoordinatesLocal(dm, &v));
1388     PetscCall(VecGetBlockSize(v, &cDim));
1389     PetscCall(VecGetLocalSize(v, &n));
1390     PetscCall(VecGetArray(v, &x));
1391     x += cDim;
1392     for (i = 0; i < n; i += cDim) x[i] += lower[2];
1393     PetscCall(VecRestoreArray(v, &x));
1394     PetscCall(DMSetCoordinatesLocal(dm, v));
1395   }
1396   PetscFunctionReturn(0);
1397 }
1398 
1399 /*@
1400   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tesselating the (x,y) plane and extruding in the third direction using wedge cells.
1401 
1402   Collective
1403 
1404   Input Parameters:
1405 + comm        - The communicator for the DM object
1406 . faces       - Number of faces per dimension, or NULL for (1, 1, 1)
1407 . lower       - The lower left corner, or NULL for (0, 0, 0)
1408 . upper       - The upper right corner, or NULL for (1, 1, 1)
1409 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
1410 . orderHeight - If PETSC_TRUE, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1411 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1412 
1413   Output Parameter:
1414 . dm  - The DM object
1415 
1416   Level: beginner
1417 
1418 .seealso: `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateWedgeCylinderMesh()`, `DMExtrude()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1419 @*/
1420 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool orderHeight, PetscBool interpolate, DM *dm)
1421 {
1422   PetscInt       fac[3] = {1, 1, 1};
1423   PetscReal      low[3] = {0, 0, 0};
1424   PetscReal      upp[3] = {1, 1, 1};
1425   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1426 
1427   PetscFunctionBegin;
1428   PetscCall(DMCreate(comm, dm));
1429   PetscCall(DMSetType(*dm, DMPLEX));
1430   PetscCall(DMPlexCreateWedgeBoxMesh_Internal(*dm, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt));
1431   if (!interpolate) {
1432     DM udm;
1433 
1434     PetscCall(DMPlexUninterpolate(*dm, &udm));
1435     PetscCall(DMPlexReplace_Internal(*dm, &udm));
1436   }
1437   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1438   PetscFunctionReturn(0);
1439 }
1440 
1441 /*@C
1442   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all DM options in the database.
1443 
1444   Logically Collective on dm
1445 
1446   Input Parameters:
1447 + dm - the DM context
1448 - prefix - the prefix to prepend to all option names
1449 
1450   Notes:
1451   A hyphen (-) must NOT be given at the beginning of the prefix name.
1452   The first character of all runtime options is AUTOMATICALLY the hyphen.
1453 
1454   Level: advanced
1455 
1456 .seealso: `SNESSetFromOptions()`
1457 @*/
1458 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
1459 {
1460   DM_Plex *mesh = (DM_Plex *)dm->data;
1461 
1462   PetscFunctionBegin;
1463   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1464   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, prefix));
1465   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)mesh->partitioner, prefix));
1466   PetscFunctionReturn(0);
1467 }
1468 
1469 /* Remap geometry to cylinder
1470    TODO: This only works for a single refinement, then it is broken
1471 
1472      Interior square: Linear interpolation is correct
1473      The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
1474      such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
1475 
1476        phi     = arctan(y/x)
1477        d_close = sqrt(1/8 + 1/4 sin^2(phi))
1478        d_far   = sqrt(1/2 + sin^2(phi))
1479 
1480      so we remap them using
1481 
1482        x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
1483        y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
1484 
1485      If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
1486 */
1487 static void snapToCylinder(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
1488 {
1489   const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1490   const PetscReal ds2 = 0.5 * dis;
1491 
1492   if ((PetscAbsScalar(u[0]) <= ds2) && (PetscAbsScalar(u[1]) <= ds2)) {
1493     f0[0] = u[0];
1494     f0[1] = u[1];
1495   } else {
1496     PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
1497 
1498     x    = PetscRealPart(u[0]);
1499     y    = PetscRealPart(u[1]);
1500     phi  = PetscAtan2Real(y, x);
1501     sinp = PetscSinReal(phi);
1502     cosp = PetscCosReal(phi);
1503     if ((PetscAbsReal(phi) > PETSC_PI / 4.0) && (PetscAbsReal(phi) < 3.0 * PETSC_PI / 4.0)) {
1504       dc = PetscAbsReal(ds2 / sinp);
1505       df = PetscAbsReal(dis / sinp);
1506       xc = ds2 * x / PetscAbsReal(y);
1507       yc = ds2 * PetscSignReal(y);
1508     } else {
1509       dc = PetscAbsReal(ds2 / cosp);
1510       df = PetscAbsReal(dis / cosp);
1511       xc = ds2 * PetscSignReal(x);
1512       yc = ds2 * y / PetscAbsReal(x);
1513     }
1514     f0[0] = xc + (u[0] - xc) * (1.0 - dc) / (df - dc);
1515     f0[1] = yc + (u[1] - yc) * (1.0 - dc) / (df - dc);
1516   }
1517   f0[2] = u[2];
1518 }
1519 
1520 static PetscErrorCode DMPlexCreateHexCylinderMesh_Internal(DM dm, DMBoundaryType periodicZ)
1521 {
1522   const PetscInt dim = 3;
1523   PetscInt       numCells, numVertices;
1524   PetscMPIInt    rank;
1525 
1526   PetscFunctionBegin;
1527   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1528   PetscCall(DMSetDimension(dm, dim));
1529   /* Create topology */
1530   {
1531     PetscInt cone[8], c;
1532 
1533     numCells    = rank == 0 ? 5 : 0;
1534     numVertices = rank == 0 ? 16 : 0;
1535     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1536       numCells *= 3;
1537       numVertices = rank == 0 ? 24 : 0;
1538     }
1539     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1540     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 8));
1541     PetscCall(DMSetUp(dm));
1542     if (rank == 0) {
1543       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1544         cone[0] = 15;
1545         cone[1] = 18;
1546         cone[2] = 17;
1547         cone[3] = 16;
1548         cone[4] = 31;
1549         cone[5] = 32;
1550         cone[6] = 33;
1551         cone[7] = 34;
1552         PetscCall(DMPlexSetCone(dm, 0, cone));
1553         cone[0] = 16;
1554         cone[1] = 17;
1555         cone[2] = 24;
1556         cone[3] = 23;
1557         cone[4] = 32;
1558         cone[5] = 36;
1559         cone[6] = 37;
1560         cone[7] = 33; /* 22 25 26 21 */
1561         PetscCall(DMPlexSetCone(dm, 1, cone));
1562         cone[0] = 18;
1563         cone[1] = 27;
1564         cone[2] = 24;
1565         cone[3] = 17;
1566         cone[4] = 34;
1567         cone[5] = 33;
1568         cone[6] = 37;
1569         cone[7] = 38;
1570         PetscCall(DMPlexSetCone(dm, 2, cone));
1571         cone[0] = 29;
1572         cone[1] = 27;
1573         cone[2] = 18;
1574         cone[3] = 15;
1575         cone[4] = 35;
1576         cone[5] = 31;
1577         cone[6] = 34;
1578         cone[7] = 38;
1579         PetscCall(DMPlexSetCone(dm, 3, cone));
1580         cone[0] = 29;
1581         cone[1] = 15;
1582         cone[2] = 16;
1583         cone[3] = 23;
1584         cone[4] = 35;
1585         cone[5] = 36;
1586         cone[6] = 32;
1587         cone[7] = 31;
1588         PetscCall(DMPlexSetCone(dm, 4, cone));
1589 
1590         cone[0] = 31;
1591         cone[1] = 34;
1592         cone[2] = 33;
1593         cone[3] = 32;
1594         cone[4] = 19;
1595         cone[5] = 22;
1596         cone[6] = 21;
1597         cone[7] = 20;
1598         PetscCall(DMPlexSetCone(dm, 5, cone));
1599         cone[0] = 32;
1600         cone[1] = 33;
1601         cone[2] = 37;
1602         cone[3] = 36;
1603         cone[4] = 22;
1604         cone[5] = 25;
1605         cone[6] = 26;
1606         cone[7] = 21;
1607         PetscCall(DMPlexSetCone(dm, 6, cone));
1608         cone[0] = 34;
1609         cone[1] = 38;
1610         cone[2] = 37;
1611         cone[3] = 33;
1612         cone[4] = 20;
1613         cone[5] = 21;
1614         cone[6] = 26;
1615         cone[7] = 28;
1616         PetscCall(DMPlexSetCone(dm, 7, cone));
1617         cone[0] = 35;
1618         cone[1] = 38;
1619         cone[2] = 34;
1620         cone[3] = 31;
1621         cone[4] = 30;
1622         cone[5] = 19;
1623         cone[6] = 20;
1624         cone[7] = 28;
1625         PetscCall(DMPlexSetCone(dm, 8, cone));
1626         cone[0] = 35;
1627         cone[1] = 31;
1628         cone[2] = 32;
1629         cone[3] = 36;
1630         cone[4] = 30;
1631         cone[5] = 25;
1632         cone[6] = 22;
1633         cone[7] = 19;
1634         PetscCall(DMPlexSetCone(dm, 9, cone));
1635 
1636         cone[0] = 19;
1637         cone[1] = 20;
1638         cone[2] = 21;
1639         cone[3] = 22;
1640         cone[4] = 15;
1641         cone[5] = 16;
1642         cone[6] = 17;
1643         cone[7] = 18;
1644         PetscCall(DMPlexSetCone(dm, 10, cone));
1645         cone[0] = 22;
1646         cone[1] = 21;
1647         cone[2] = 26;
1648         cone[3] = 25;
1649         cone[4] = 16;
1650         cone[5] = 23;
1651         cone[6] = 24;
1652         cone[7] = 17;
1653         PetscCall(DMPlexSetCone(dm, 11, cone));
1654         cone[0] = 20;
1655         cone[1] = 28;
1656         cone[2] = 26;
1657         cone[3] = 21;
1658         cone[4] = 18;
1659         cone[5] = 17;
1660         cone[6] = 24;
1661         cone[7] = 27;
1662         PetscCall(DMPlexSetCone(dm, 12, cone));
1663         cone[0] = 30;
1664         cone[1] = 28;
1665         cone[2] = 20;
1666         cone[3] = 19;
1667         cone[4] = 29;
1668         cone[5] = 15;
1669         cone[6] = 18;
1670         cone[7] = 27;
1671         PetscCall(DMPlexSetCone(dm, 13, cone));
1672         cone[0] = 30;
1673         cone[1] = 19;
1674         cone[2] = 22;
1675         cone[3] = 25;
1676         cone[4] = 29;
1677         cone[5] = 23;
1678         cone[6] = 16;
1679         cone[7] = 15;
1680         PetscCall(DMPlexSetCone(dm, 14, cone));
1681       } else {
1682         cone[0] = 5;
1683         cone[1] = 8;
1684         cone[2] = 7;
1685         cone[3] = 6;
1686         cone[4] = 9;
1687         cone[5] = 12;
1688         cone[6] = 11;
1689         cone[7] = 10;
1690         PetscCall(DMPlexSetCone(dm, 0, cone));
1691         cone[0] = 6;
1692         cone[1] = 7;
1693         cone[2] = 14;
1694         cone[3] = 13;
1695         cone[4] = 12;
1696         cone[5] = 15;
1697         cone[6] = 16;
1698         cone[7] = 11;
1699         PetscCall(DMPlexSetCone(dm, 1, cone));
1700         cone[0] = 8;
1701         cone[1] = 17;
1702         cone[2] = 14;
1703         cone[3] = 7;
1704         cone[4] = 10;
1705         cone[5] = 11;
1706         cone[6] = 16;
1707         cone[7] = 18;
1708         PetscCall(DMPlexSetCone(dm, 2, cone));
1709         cone[0] = 19;
1710         cone[1] = 17;
1711         cone[2] = 8;
1712         cone[3] = 5;
1713         cone[4] = 20;
1714         cone[5] = 9;
1715         cone[6] = 10;
1716         cone[7] = 18;
1717         PetscCall(DMPlexSetCone(dm, 3, cone));
1718         cone[0] = 19;
1719         cone[1] = 5;
1720         cone[2] = 6;
1721         cone[3] = 13;
1722         cone[4] = 20;
1723         cone[5] = 15;
1724         cone[6] = 12;
1725         cone[7] = 9;
1726         PetscCall(DMPlexSetCone(dm, 4, cone));
1727       }
1728     }
1729     PetscCall(DMPlexSymmetrize(dm));
1730     PetscCall(DMPlexStratify(dm));
1731   }
1732   /* Create cube geometry */
1733   {
1734     Vec             coordinates;
1735     PetscSection    coordSection;
1736     PetscScalar    *coords;
1737     PetscInt        coordSize, v;
1738     const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1739     const PetscReal ds2 = dis / 2.0;
1740 
1741     /* Build coordinates */
1742     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1743     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1744     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1745     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1746     for (v = numCells; v < numCells + numVertices; ++v) {
1747       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1748       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1749     }
1750     PetscCall(PetscSectionSetUp(coordSection));
1751     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1752     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1753     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1754     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1755     PetscCall(VecSetBlockSize(coordinates, dim));
1756     PetscCall(VecSetType(coordinates, VECSTANDARD));
1757     PetscCall(VecGetArray(coordinates, &coords));
1758     if (rank == 0) {
1759       coords[0 * dim + 0]  = -ds2;
1760       coords[0 * dim + 1]  = -ds2;
1761       coords[0 * dim + 2]  = 0.0;
1762       coords[1 * dim + 0]  = ds2;
1763       coords[1 * dim + 1]  = -ds2;
1764       coords[1 * dim + 2]  = 0.0;
1765       coords[2 * dim + 0]  = ds2;
1766       coords[2 * dim + 1]  = ds2;
1767       coords[2 * dim + 2]  = 0.0;
1768       coords[3 * dim + 0]  = -ds2;
1769       coords[3 * dim + 1]  = ds2;
1770       coords[3 * dim + 2]  = 0.0;
1771       coords[4 * dim + 0]  = -ds2;
1772       coords[4 * dim + 1]  = -ds2;
1773       coords[4 * dim + 2]  = 1.0;
1774       coords[5 * dim + 0]  = -ds2;
1775       coords[5 * dim + 1]  = ds2;
1776       coords[5 * dim + 2]  = 1.0;
1777       coords[6 * dim + 0]  = ds2;
1778       coords[6 * dim + 1]  = ds2;
1779       coords[6 * dim + 2]  = 1.0;
1780       coords[7 * dim + 0]  = ds2;
1781       coords[7 * dim + 1]  = -ds2;
1782       coords[7 * dim + 2]  = 1.0;
1783       coords[8 * dim + 0]  = dis;
1784       coords[8 * dim + 1]  = -dis;
1785       coords[8 * dim + 2]  = 0.0;
1786       coords[9 * dim + 0]  = dis;
1787       coords[9 * dim + 1]  = dis;
1788       coords[9 * dim + 2]  = 0.0;
1789       coords[10 * dim + 0] = dis;
1790       coords[10 * dim + 1] = -dis;
1791       coords[10 * dim + 2] = 1.0;
1792       coords[11 * dim + 0] = dis;
1793       coords[11 * dim + 1] = dis;
1794       coords[11 * dim + 2] = 1.0;
1795       coords[12 * dim + 0] = -dis;
1796       coords[12 * dim + 1] = dis;
1797       coords[12 * dim + 2] = 0.0;
1798       coords[13 * dim + 0] = -dis;
1799       coords[13 * dim + 1] = dis;
1800       coords[13 * dim + 2] = 1.0;
1801       coords[14 * dim + 0] = -dis;
1802       coords[14 * dim + 1] = -dis;
1803       coords[14 * dim + 2] = 0.0;
1804       coords[15 * dim + 0] = -dis;
1805       coords[15 * dim + 1] = -dis;
1806       coords[15 * dim + 2] = 1.0;
1807       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1808         /* 15 31 19 */ coords[16 * dim + 0] = -ds2;
1809         coords[16 * dim + 1]                = -ds2;
1810         coords[16 * dim + 2]                = 0.5;
1811         /* 16 32 22 */ coords[17 * dim + 0] = ds2;
1812         coords[17 * dim + 1]                = -ds2;
1813         coords[17 * dim + 2]                = 0.5;
1814         /* 17 33 21 */ coords[18 * dim + 0] = ds2;
1815         coords[18 * dim + 1]                = ds2;
1816         coords[18 * dim + 2]                = 0.5;
1817         /* 18 34 20 */ coords[19 * dim + 0] = -ds2;
1818         coords[19 * dim + 1]                = ds2;
1819         coords[19 * dim + 2]                = 0.5;
1820         /* 29 35 30 */ coords[20 * dim + 0] = -dis;
1821         coords[20 * dim + 1]                = -dis;
1822         coords[20 * dim + 2]                = 0.5;
1823         /* 23 36 25 */ coords[21 * dim + 0] = dis;
1824         coords[21 * dim + 1]                = -dis;
1825         coords[21 * dim + 2]                = 0.5;
1826         /* 24 37 26 */ coords[22 * dim + 0] = dis;
1827         coords[22 * dim + 1]                = dis;
1828         coords[22 * dim + 2]                = 0.5;
1829         /* 27 38 28 */ coords[23 * dim + 0] = -dis;
1830         coords[23 * dim + 1]                = dis;
1831         coords[23 * dim + 2]                = 0.5;
1832       }
1833     }
1834     PetscCall(VecRestoreArray(coordinates, &coords));
1835     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1836     PetscCall(VecDestroy(&coordinates));
1837   }
1838   /* Create periodicity */
1839   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1840     PetscReal L[3]       = {-1., -1., 0.};
1841     PetscReal maxCell[3] = {-1., -1., 0.};
1842     PetscReal lower[3]   = {0.0, 0.0, 0.0};
1843     PetscReal upper[3]   = {1.0, 1.0, 1.5};
1844     PetscInt  numZCells  = 3;
1845 
1846     L[2]       = upper[2] - lower[2];
1847     maxCell[2] = 1.1 * (L[2] / numZCells);
1848     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1849   }
1850   {
1851     DM          cdm;
1852     PetscDS     cds;
1853     PetscScalar c[2] = {1.0, 1.0};
1854 
1855     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToCylinder));
1856     PetscCall(DMGetCoordinateDM(dm, &cdm));
1857     PetscCall(DMGetDS(cdm, &cds));
1858     PetscCall(PetscDSSetConstants(cds, 2, c));
1859   }
1860   /* Wait for coordinate creation before doing in-place modification */
1861   PetscCall(DMPlexInterpolateInPlace_Internal(dm));
1862   PetscFunctionReturn(0);
1863 }
1864 
1865 /*@
1866   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
1867 
1868   Collective
1869 
1870   Input Parameters:
1871 + comm      - The communicator for the DM object
1872 - periodicZ - The boundary type for the Z direction
1873 
1874   Output Parameter:
1875 . dm  - The DM object
1876 
1877   Note:
1878   Here is the output numbering looking from the bottom of the cylinder:
1879 $       17-----14
1880 $        |     |
1881 $        |  2  |
1882 $        |     |
1883 $ 17-----8-----7-----14
1884 $  |     |     |     |
1885 $  |  3  |  0  |  1  |
1886 $  |     |     |     |
1887 $ 19-----5-----6-----13
1888 $        |     |
1889 $        |  4  |
1890 $        |     |
1891 $       19-----13
1892 $
1893 $ and up through the top
1894 $
1895 $       18-----16
1896 $        |     |
1897 $        |  2  |
1898 $        |     |
1899 $ 18----10----11-----16
1900 $  |     |     |     |
1901 $  |  3  |  0  |  1  |
1902 $  |     |     |     |
1903 $ 20-----9----12-----15
1904 $        |     |
1905 $        |  4  |
1906 $        |     |
1907 $       20-----15
1908 
1909   Level: beginner
1910 
1911 .seealso: `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1912 @*/
1913 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, DMBoundaryType periodicZ, DM *dm)
1914 {
1915   PetscFunctionBegin;
1916   PetscValidPointer(dm, 3);
1917   PetscCall(DMCreate(comm, dm));
1918   PetscCall(DMSetType(*dm, DMPLEX));
1919   PetscCall(DMPlexCreateHexCylinderMesh_Internal(*dm, periodicZ));
1920   PetscFunctionReturn(0);
1921 }
1922 
1923 static PetscErrorCode DMPlexCreateWedgeCylinderMesh_Internal(DM dm, PetscInt n, PetscBool interpolate)
1924 {
1925   const PetscInt dim = 3;
1926   PetscInt       numCells, numVertices, v;
1927   PetscMPIInt    rank;
1928 
1929   PetscFunctionBegin;
1930   PetscCheck(n >= 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %" PetscInt_FMT " cannot be negative", n);
1931   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1932   PetscCall(DMSetDimension(dm, dim));
1933   /* Must create the celltype label here so that we do not automatically try to compute the types */
1934   PetscCall(DMCreateLabel(dm, "celltype"));
1935   /* Create topology */
1936   {
1937     PetscInt cone[6], c;
1938 
1939     numCells    = rank == 0 ? n : 0;
1940     numVertices = rank == 0 ? 2 * (n + 1) : 0;
1941     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1942     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
1943     PetscCall(DMSetUp(dm));
1944     for (c = 0; c < numCells; c++) {
1945       cone[0] = c + n * 1;
1946       cone[1] = (c + 1) % n + n * 1;
1947       cone[2] = 0 + 3 * n;
1948       cone[3] = c + n * 2;
1949       cone[4] = (c + 1) % n + n * 2;
1950       cone[5] = 1 + 3 * n;
1951       PetscCall(DMPlexSetCone(dm, c, cone));
1952       PetscCall(DMPlexSetCellType(dm, c, DM_POLYTOPE_TRI_PRISM_TENSOR));
1953     }
1954     PetscCall(DMPlexSymmetrize(dm));
1955     PetscCall(DMPlexStratify(dm));
1956   }
1957   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetCellType(dm, v, DM_POLYTOPE_POINT));
1958   /* Create cylinder geometry */
1959   {
1960     Vec          coordinates;
1961     PetscSection coordSection;
1962     PetscScalar *coords;
1963     PetscInt     coordSize, c;
1964 
1965     /* Build coordinates */
1966     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1967     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1968     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1969     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1970     for (v = numCells; v < numCells + numVertices; ++v) {
1971       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1972       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1973     }
1974     PetscCall(PetscSectionSetUp(coordSection));
1975     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1976     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1977     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1978     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1979     PetscCall(VecSetBlockSize(coordinates, dim));
1980     PetscCall(VecSetType(coordinates, VECSTANDARD));
1981     PetscCall(VecGetArray(coordinates, &coords));
1982     for (c = 0; c < numCells; c++) {
1983       coords[(c + 0 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
1984       coords[(c + 0 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
1985       coords[(c + 0 * n) * dim + 2] = 1.0;
1986       coords[(c + 1 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
1987       coords[(c + 1 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
1988       coords[(c + 1 * n) * dim + 2] = 0.0;
1989     }
1990     if (rank == 0) {
1991       coords[(2 * n + 0) * dim + 0] = 0.0;
1992       coords[(2 * n + 0) * dim + 1] = 0.0;
1993       coords[(2 * n + 0) * dim + 2] = 1.0;
1994       coords[(2 * n + 1) * dim + 0] = 0.0;
1995       coords[(2 * n + 1) * dim + 1] = 0.0;
1996       coords[(2 * n + 1) * dim + 2] = 0.0;
1997     }
1998     PetscCall(VecRestoreArray(coordinates, &coords));
1999     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2000     PetscCall(VecDestroy(&coordinates));
2001   }
2002   /* Interpolate */
2003   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2004   PetscFunctionReturn(0);
2005 }
2006 
2007 /*@
2008   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
2009 
2010   Collective
2011 
2012   Input Parameters:
2013 + comm - The communicator for the DM object
2014 . n    - The number of wedges around the origin
2015 - interpolate - Create edges and faces
2016 
2017   Output Parameter:
2018 . dm  - The DM object
2019 
2020   Level: beginner
2021 
2022 .seealso: `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2023 @*/
2024 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
2025 {
2026   PetscFunctionBegin;
2027   PetscValidPointer(dm, 4);
2028   PetscCall(DMCreate(comm, dm));
2029   PetscCall(DMSetType(*dm, DMPLEX));
2030   PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(*dm, n, interpolate));
2031   PetscFunctionReturn(0);
2032 }
2033 
2034 static inline PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2035 {
2036   PetscReal prod = 0.0;
2037   PetscInt  i;
2038   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
2039   return PetscSqrtReal(prod);
2040 }
2041 static inline PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2042 {
2043   PetscReal prod = 0.0;
2044   PetscInt  i;
2045   for (i = 0; i < dim; ++i) prod += x[i] * y[i];
2046   return prod;
2047 }
2048 
2049 /* The first constant is the sphere radius */
2050 static void snapToSphere(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
2051 {
2052   PetscReal r     = PetscRealPart(constants[0]);
2053   PetscReal norm2 = 0.0, fac;
2054   PetscInt  n     = uOff[1] - uOff[0], d;
2055 
2056   for (d = 0; d < n; ++d) norm2 += PetscSqr(PetscRealPart(u[d]));
2057   fac = r / PetscSqrtReal(norm2);
2058   for (d = 0; d < n; ++d) f0[d] = u[d] * fac;
2059 }
2060 
2061 static PetscErrorCode DMPlexCreateSphereMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, PetscReal R)
2062 {
2063   const PetscInt embedDim = dim + 1;
2064   PetscSection   coordSection;
2065   Vec            coordinates;
2066   PetscScalar   *coords;
2067   PetscReal     *coordsIn;
2068   PetscInt       numCells, numEdges, numVerts = 0, firstVertex = 0, v, firstEdge, coordSize, d, c, e;
2069   PetscMPIInt    rank;
2070 
2071   PetscFunctionBegin;
2072   PetscValidLogicalCollectiveBool(dm, simplex, 3);
2073   PetscCall(DMSetDimension(dm, dim));
2074   PetscCall(DMSetCoordinateDim(dm, dim + 1));
2075   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2076   switch (dim) {
2077   case 2:
2078     if (simplex) {
2079       const PetscReal radius    = PetscSqrtReal(1 + PETSC_PHI * PETSC_PHI) / (1.0 + PETSC_PHI);
2080       const PetscReal edgeLen   = 2.0 / (1.0 + PETSC_PHI) * (R / radius);
2081       const PetscInt  degree    = 5;
2082       PetscReal       vertex[3] = {0.0, 1.0 / (1.0 + PETSC_PHI), PETSC_PHI / (1.0 + PETSC_PHI)};
2083       PetscInt        s[3]      = {1, 1, 1};
2084       PetscInt        cone[3];
2085       PetscInt       *graph, p, i, j, k;
2086 
2087       vertex[0] *= R / radius;
2088       vertex[1] *= R / radius;
2089       vertex[2] *= R / radius;
2090       numCells    = rank == 0 ? 20 : 0;
2091       numVerts    = rank == 0 ? 12 : 0;
2092       firstVertex = numCells;
2093       /* Use icosahedron, which for a R-sphere has coordinates which are all cyclic permutations of
2094 
2095            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
2096 
2097          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2098          length is then given by 2/(1+\phi) = 2 * 0.38197 = 0.76393.
2099       */
2100       /* Construct vertices */
2101       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2102       if (rank == 0) {
2103         for (p = 0, i = 0; p < embedDim; ++p) {
2104           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2105             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2106               for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertex[(d + p) % embedDim];
2107               ++i;
2108             }
2109           }
2110         }
2111       }
2112       /* Construct graph */
2113       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2114       for (i = 0; i < numVerts; ++i) {
2115         for (j = 0, k = 0; j < numVerts; ++j) {
2116           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2117             graph[i * numVerts + j] = 1;
2118             ++k;
2119           }
2120         }
2121         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid icosahedron, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2122       }
2123       /* Build Topology */
2124       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2125       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2126       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2127       /* Cells */
2128       for (i = 0, c = 0; i < numVerts; ++i) {
2129         for (j = 0; j < i; ++j) {
2130           for (k = 0; k < j; ++k) {
2131             if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i]) {
2132               cone[0] = firstVertex + i;
2133               cone[1] = firstVertex + j;
2134               cone[2] = firstVertex + k;
2135               /* Check orientation */
2136               {
2137                 const PetscInt epsilon[3][3][3] = {
2138                   {{0, 0, 0},  {0, 0, 1},  {0, -1, 0}},
2139                   {{0, 0, -1}, {0, 0, 0},  {1, 0, 0} },
2140                   {{0, 1, 0},  {-1, 0, 0}, {0, 0, 0} }
2141                 };
2142                 PetscReal normal[3];
2143                 PetscInt  e, f;
2144 
2145                 for (d = 0; d < embedDim; ++d) {
2146                   normal[d] = 0.0;
2147                   for (e = 0; e < embedDim; ++e) {
2148                     for (f = 0; f < embedDim; ++f) normal[d] += epsilon[d][e][f] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]);
2149                   }
2150                 }
2151                 if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2152                   PetscInt tmp = cone[1];
2153                   cone[1]      = cone[2];
2154                   cone[2]      = tmp;
2155                 }
2156               }
2157               PetscCall(DMPlexSetCone(dm, c++, cone));
2158             }
2159           }
2160         }
2161       }
2162       PetscCall(DMPlexSymmetrize(dm));
2163       PetscCall(DMPlexStratify(dm));
2164       PetscCall(PetscFree(graph));
2165     } else {
2166       /*
2167         12-21--13
2168          |     |
2169         25  4  24
2170          |     |
2171   12-25--9-16--8-24--13
2172    |     |     |     |
2173   23  5 17  0 15  3  22
2174    |     |     |     |
2175   10-20--6-14--7-19--11
2176          |     |
2177         20  1  19
2178          |     |
2179         10-18--11
2180          |     |
2181         23  2  22
2182          |     |
2183         12-21--13
2184        */
2185       PetscInt cone[4], ornt[4];
2186 
2187       numCells    = rank == 0 ? 6 : 0;
2188       numEdges    = rank == 0 ? 12 : 0;
2189       numVerts    = rank == 0 ? 8 : 0;
2190       firstVertex = numCells;
2191       firstEdge   = numCells + numVerts;
2192       /* Build Topology */
2193       PetscCall(DMPlexSetChart(dm, 0, numCells + numEdges + numVerts));
2194       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 4));
2195       for (e = firstEdge; e < firstEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
2196       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2197       if (rank == 0) {
2198         /* Cell 0 */
2199         cone[0] = 14;
2200         cone[1] = 15;
2201         cone[2] = 16;
2202         cone[3] = 17;
2203         PetscCall(DMPlexSetCone(dm, 0, cone));
2204         ornt[0] = 0;
2205         ornt[1] = 0;
2206         ornt[2] = 0;
2207         ornt[3] = 0;
2208         PetscCall(DMPlexSetConeOrientation(dm, 0, ornt));
2209         /* Cell 1 */
2210         cone[0] = 18;
2211         cone[1] = 19;
2212         cone[2] = 14;
2213         cone[3] = 20;
2214         PetscCall(DMPlexSetCone(dm, 1, cone));
2215         ornt[0] = 0;
2216         ornt[1] = 0;
2217         ornt[2] = -1;
2218         ornt[3] = 0;
2219         PetscCall(DMPlexSetConeOrientation(dm, 1, ornt));
2220         /* Cell 2 */
2221         cone[0] = 21;
2222         cone[1] = 22;
2223         cone[2] = 18;
2224         cone[3] = 23;
2225         PetscCall(DMPlexSetCone(dm, 2, cone));
2226         ornt[0] = 0;
2227         ornt[1] = 0;
2228         ornt[2] = -1;
2229         ornt[3] = 0;
2230         PetscCall(DMPlexSetConeOrientation(dm, 2, ornt));
2231         /* Cell 3 */
2232         cone[0] = 19;
2233         cone[1] = 22;
2234         cone[2] = 24;
2235         cone[3] = 15;
2236         PetscCall(DMPlexSetCone(dm, 3, cone));
2237         ornt[0] = -1;
2238         ornt[1] = -1;
2239         ornt[2] = 0;
2240         ornt[3] = -1;
2241         PetscCall(DMPlexSetConeOrientation(dm, 3, ornt));
2242         /* Cell 4 */
2243         cone[0] = 16;
2244         cone[1] = 24;
2245         cone[2] = 21;
2246         cone[3] = 25;
2247         PetscCall(DMPlexSetCone(dm, 4, cone));
2248         ornt[0] = -1;
2249         ornt[1] = -1;
2250         ornt[2] = -1;
2251         ornt[3] = 0;
2252         PetscCall(DMPlexSetConeOrientation(dm, 4, ornt));
2253         /* Cell 5 */
2254         cone[0] = 20;
2255         cone[1] = 17;
2256         cone[2] = 25;
2257         cone[3] = 23;
2258         PetscCall(DMPlexSetCone(dm, 5, cone));
2259         ornt[0] = -1;
2260         ornt[1] = -1;
2261         ornt[2] = -1;
2262         ornt[3] = -1;
2263         PetscCall(DMPlexSetConeOrientation(dm, 5, ornt));
2264         /* Edges */
2265         cone[0] = 6;
2266         cone[1] = 7;
2267         PetscCall(DMPlexSetCone(dm, 14, cone));
2268         cone[0] = 7;
2269         cone[1] = 8;
2270         PetscCall(DMPlexSetCone(dm, 15, cone));
2271         cone[0] = 8;
2272         cone[1] = 9;
2273         PetscCall(DMPlexSetCone(dm, 16, cone));
2274         cone[0] = 9;
2275         cone[1] = 6;
2276         PetscCall(DMPlexSetCone(dm, 17, cone));
2277         cone[0] = 10;
2278         cone[1] = 11;
2279         PetscCall(DMPlexSetCone(dm, 18, cone));
2280         cone[0] = 11;
2281         cone[1] = 7;
2282         PetscCall(DMPlexSetCone(dm, 19, cone));
2283         cone[0] = 6;
2284         cone[1] = 10;
2285         PetscCall(DMPlexSetCone(dm, 20, cone));
2286         cone[0] = 12;
2287         cone[1] = 13;
2288         PetscCall(DMPlexSetCone(dm, 21, cone));
2289         cone[0] = 13;
2290         cone[1] = 11;
2291         PetscCall(DMPlexSetCone(dm, 22, cone));
2292         cone[0] = 10;
2293         cone[1] = 12;
2294         PetscCall(DMPlexSetCone(dm, 23, cone));
2295         cone[0] = 13;
2296         cone[1] = 8;
2297         PetscCall(DMPlexSetCone(dm, 24, cone));
2298         cone[0] = 12;
2299         cone[1] = 9;
2300         PetscCall(DMPlexSetCone(dm, 25, cone));
2301       }
2302       PetscCall(DMPlexSymmetrize(dm));
2303       PetscCall(DMPlexStratify(dm));
2304       /* Build coordinates */
2305       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2306       if (rank == 0) {
2307         coordsIn[0 * embedDim + 0] = -R;
2308         coordsIn[0 * embedDim + 1] = R;
2309         coordsIn[0 * embedDim + 2] = -R;
2310         coordsIn[1 * embedDim + 0] = R;
2311         coordsIn[1 * embedDim + 1] = R;
2312         coordsIn[1 * embedDim + 2] = -R;
2313         coordsIn[2 * embedDim + 0] = R;
2314         coordsIn[2 * embedDim + 1] = -R;
2315         coordsIn[2 * embedDim + 2] = -R;
2316         coordsIn[3 * embedDim + 0] = -R;
2317         coordsIn[3 * embedDim + 1] = -R;
2318         coordsIn[3 * embedDim + 2] = -R;
2319         coordsIn[4 * embedDim + 0] = -R;
2320         coordsIn[4 * embedDim + 1] = R;
2321         coordsIn[4 * embedDim + 2] = R;
2322         coordsIn[5 * embedDim + 0] = R;
2323         coordsIn[5 * embedDim + 1] = R;
2324         coordsIn[5 * embedDim + 2] = R;
2325         coordsIn[6 * embedDim + 0] = -R;
2326         coordsIn[6 * embedDim + 1] = -R;
2327         coordsIn[6 * embedDim + 2] = R;
2328         coordsIn[7 * embedDim + 0] = R;
2329         coordsIn[7 * embedDim + 1] = -R;
2330         coordsIn[7 * embedDim + 2] = R;
2331       }
2332     }
2333     break;
2334   case 3:
2335     if (simplex) {
2336       const PetscReal edgeLen         = 1.0 / PETSC_PHI;
2337       PetscReal       vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
2338       PetscReal       vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
2339       PetscReal       vertexC[4]      = {0.5, 0.5 * PETSC_PHI, 0.5 / PETSC_PHI, 0.0};
2340       const PetscInt  degree          = 12;
2341       PetscInt        s[4]            = {1, 1, 1};
2342       PetscInt        evenPerm[12][4] = {
2343         {0, 1, 2, 3},
2344         {0, 2, 3, 1},
2345         {0, 3, 1, 2},
2346         {1, 0, 3, 2},
2347         {1, 2, 0, 3},
2348         {1, 3, 2, 0},
2349         {2, 0, 1, 3},
2350         {2, 1, 3, 0},
2351         {2, 3, 0, 1},
2352         {3, 0, 2, 1},
2353         {3, 1, 0, 2},
2354         {3, 2, 1, 0}
2355       };
2356       PetscInt  cone[4];
2357       PetscInt *graph, p, i, j, k, l;
2358 
2359       vertexA[0] *= R;
2360       vertexA[1] *= R;
2361       vertexA[2] *= R;
2362       vertexA[3] *= R;
2363       vertexB[0] *= R;
2364       vertexB[1] *= R;
2365       vertexB[2] *= R;
2366       vertexB[3] *= R;
2367       vertexC[0] *= R;
2368       vertexC[1] *= R;
2369       vertexC[2] *= R;
2370       vertexC[3] *= R;
2371       numCells    = rank == 0 ? 600 : 0;
2372       numVerts    = rank == 0 ? 120 : 0;
2373       firstVertex = numCells;
2374       /* Use the 600-cell, which for a unit sphere has coordinates which are
2375 
2376            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
2377                (\pm 1,    0,       0,      0)  all cyclic permutations   8
2378            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
2379 
2380          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2381          length is then given by 1/\phi = 0.61803.
2382 
2383          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
2384          http://mathworld.wolfram.com/600-Cell.html
2385       */
2386       /* Construct vertices */
2387       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2388       i = 0;
2389       if (rank == 0) {
2390         for (s[0] = -1; s[0] < 2; s[0] += 2) {
2391           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2392             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2393               for (s[3] = -1; s[3] < 2; s[3] += 2) {
2394                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[d] * vertexA[d];
2395                 ++i;
2396               }
2397             }
2398           }
2399         }
2400         for (p = 0; p < embedDim; ++p) {
2401           s[1] = s[2] = s[3] = 1;
2402           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2403             for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertexB[(d + p) % embedDim];
2404             ++i;
2405           }
2406         }
2407         for (p = 0; p < 12; ++p) {
2408           s[3] = 1;
2409           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2410             for (s[1] = -1; s[1] < 2; s[1] += 2) {
2411               for (s[2] = -1; s[2] < 2; s[2] += 2) {
2412                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[evenPerm[p][d]] * vertexC[evenPerm[p][d]];
2413                 ++i;
2414               }
2415             }
2416           }
2417         }
2418       }
2419       PetscCheck(i == numVerts, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertices %" PetscInt_FMT " != %" PetscInt_FMT, i, numVerts);
2420       /* Construct graph */
2421       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2422       for (i = 0; i < numVerts; ++i) {
2423         for (j = 0, k = 0; j < numVerts; ++j) {
2424           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2425             graph[i * numVerts + j] = 1;
2426             ++k;
2427           }
2428         }
2429         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2430       }
2431       /* Build Topology */
2432       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2433       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2434       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2435       /* Cells */
2436       if (rank == 0) {
2437         for (i = 0, c = 0; i < numVerts; ++i) {
2438           for (j = 0; j < i; ++j) {
2439             for (k = 0; k < j; ++k) {
2440               for (l = 0; l < k; ++l) {
2441                 if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i] && graph[l * numVerts + i] && graph[l * numVerts + j] && graph[l * numVerts + k]) {
2442                   cone[0] = firstVertex + i;
2443                   cone[1] = firstVertex + j;
2444                   cone[2] = firstVertex + k;
2445                   cone[3] = firstVertex + l;
2446                   /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
2447                   {
2448                     const PetscInt epsilon[4][4][4][4] = {
2449                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, -1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 0, 0}, {0, 1, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 1, 0}, {0, -1, 0, 0}, {0, 0, 0, 0}}},
2450 
2451                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}}, {{0, 0, -1, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}}},
2452 
2453                       {{{0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, 0, 0}, {0, -1, 0, 0}}, {{0, 0, 0, -1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 1, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}},
2454 
2455                       {{{0, 0, 0, 0}, {0, 0, -1, 0}, {0, 1, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 1, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}}, {{0, -1, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}} }
2456                     };
2457                     PetscReal normal[4];
2458                     PetscInt  e, f, g;
2459 
2460                     for (d = 0; d < embedDim; ++d) {
2461                       normal[d] = 0.0;
2462                       for (e = 0; e < embedDim; ++e) {
2463                         for (f = 0; f < embedDim; ++f) {
2464                           for (g = 0; g < embedDim; ++g) {
2465                             normal[d] += epsilon[d][e][f][g] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]) * (coordsIn[l * embedDim + f] - coordsIn[i * embedDim + f]);
2466                           }
2467                         }
2468                       }
2469                     }
2470                     if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2471                       PetscInt tmp = cone[1];
2472                       cone[1]      = cone[2];
2473                       cone[2]      = tmp;
2474                     }
2475                   }
2476                   PetscCall(DMPlexSetCone(dm, c++, cone));
2477                 }
2478               }
2479             }
2480           }
2481         }
2482       }
2483       PetscCall(DMPlexSymmetrize(dm));
2484       PetscCall(DMPlexStratify(dm));
2485       PetscCall(PetscFree(graph));
2486     }
2487     break;
2488   default:
2489     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unsupported dimension for sphere: %" PetscInt_FMT, dim);
2490   }
2491   /* Create coordinates */
2492   PetscCall(DMGetCoordinateSection(dm, &coordSection));
2493   PetscCall(PetscSectionSetNumFields(coordSection, 1));
2494   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, embedDim));
2495   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVerts));
2496   for (v = firstVertex; v < firstVertex + numVerts; ++v) {
2497     PetscCall(PetscSectionSetDof(coordSection, v, embedDim));
2498     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, embedDim));
2499   }
2500   PetscCall(PetscSectionSetUp(coordSection));
2501   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2502   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2503   PetscCall(VecSetBlockSize(coordinates, embedDim));
2504   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2505   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2506   PetscCall(VecSetType(coordinates, VECSTANDARD));
2507   PetscCall(VecGetArray(coordinates, &coords));
2508   for (v = 0; v < numVerts; ++v)
2509     for (d = 0; d < embedDim; ++d) coords[v * embedDim + d] = coordsIn[v * embedDim + d];
2510   PetscCall(VecRestoreArray(coordinates, &coords));
2511   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2512   PetscCall(VecDestroy(&coordinates));
2513   PetscCall(PetscFree(coordsIn));
2514   {
2515     DM          cdm;
2516     PetscDS     cds;
2517     PetscScalar c = R;
2518 
2519     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToSphere));
2520     PetscCall(DMGetCoordinateDM(dm, &cdm));
2521     PetscCall(DMGetDS(cdm, &cds));
2522     PetscCall(PetscDSSetConstants(cds, 1, &c));
2523   }
2524   /* Wait for coordinate creation before doing in-place modification */
2525   if (simplex) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2526   PetscFunctionReturn(0);
2527 }
2528 
2529 typedef void (*TPSEvaluateFunc)(const PetscReal[], PetscReal *, PetscReal[], PetscReal (*)[3]);
2530 
2531 /*
2532  The Schwarz P implicit surface is
2533 
2534      f(x) = cos(x0) + cos(x1) + cos(x2) = 0
2535 */
2536 static void TPSEvaluate_SchwarzP(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2537 {
2538   PetscReal c[3] = {PetscCosReal(y[0] * PETSC_PI), PetscCosReal(y[1] * PETSC_PI), PetscCosReal(y[2] * PETSC_PI)};
2539   PetscReal g[3] = {-PetscSinReal(y[0] * PETSC_PI), -PetscSinReal(y[1] * PETSC_PI), -PetscSinReal(y[2] * PETSC_PI)};
2540   f[0]           = c[0] + c[1] + c[2];
2541   for (PetscInt i = 0; i < 3; i++) {
2542     grad[i] = PETSC_PI * g[i];
2543     for (PetscInt j = 0; j < 3; j++) hess[i][j] = (i == j) ? -PetscSqr(PETSC_PI) * c[i] : 0.;
2544   }
2545 }
2546 
2547 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2548 static PetscErrorCode TPSExtrudeNormalFunc_SchwarzP(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2549 {
2550   for (PetscInt i = 0; i < 3; i++) u[i] = -PETSC_PI * PetscSinReal(x[i] * PETSC_PI);
2551   return 0;
2552 }
2553 
2554 /*
2555  The Gyroid implicit surface is
2556 
2557  f(x,y,z) = sin(pi * x) * cos (pi * (y + 1/2))  + sin(pi * (y + 1/2)) * cos(pi * (z + 1/4)) + sin(pi * (z + 1/4)) * cos(pi * x)
2558 
2559 */
2560 static void TPSEvaluate_Gyroid(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2561 {
2562   PetscReal s[3] = {PetscSinReal(PETSC_PI * y[0]), PetscSinReal(PETSC_PI * (y[1] + .5)), PetscSinReal(PETSC_PI * (y[2] + .25))};
2563   PetscReal c[3] = {PetscCosReal(PETSC_PI * y[0]), PetscCosReal(PETSC_PI * (y[1] + .5)), PetscCosReal(PETSC_PI * (y[2] + .25))};
2564   f[0]           = s[0] * c[1] + s[1] * c[2] + s[2] * c[0];
2565   grad[0]        = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2566   grad[1]        = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2567   grad[2]        = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2568   hess[0][0]     = -PetscSqr(PETSC_PI) * (s[0] * c[1] + s[2] * c[0]);
2569   hess[0][1]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2570   hess[0][2]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2571   hess[1][0]     = -PetscSqr(PETSC_PI) * (s[1] * c[2] + s[0] * c[1]);
2572   hess[1][1]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2573   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2574   hess[2][0]     = -PetscSqr(PETSC_PI) * (s[2] * c[0] + s[1] * c[2]);
2575   hess[2][1]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2576   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2577 }
2578 
2579 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2580 static PetscErrorCode TPSExtrudeNormalFunc_Gyroid(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2581 {
2582   PetscReal s[3] = {PetscSinReal(PETSC_PI * x[0]), PetscSinReal(PETSC_PI * (x[1] + .5)), PetscSinReal(PETSC_PI * (x[2] + .25))};
2583   PetscReal c[3] = {PetscCosReal(PETSC_PI * x[0]), PetscCosReal(PETSC_PI * (x[1] + .5)), PetscCosReal(PETSC_PI * (x[2] + .25))};
2584   u[0]           = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2585   u[1]           = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2586   u[2]           = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2587   return 0;
2588 }
2589 
2590 /*
2591    We wish to solve
2592 
2593          min_y || y - x ||^2  subject to f(y) = 0
2594 
2595    Let g(y) = grad(f).  The minimization problem is equivalent to asking to satisfy
2596    f(y) = 0 and (y-x) is parallel to g(y).  We do this by using Householder QR to obtain a basis for the
2597    tangent space and ask for both components in the tangent space to be zero.
2598 
2599    Take g to be a column vector and compute the "full QR" factorization Q R = g,
2600    where Q = I - 2 n n^T is a symmetric orthogonal matrix.
2601    The first column of Q is parallel to g so the remaining two columns span the null space.
2602    Let Qn = Q[:,1:] be those remaining columns.  Then Qn Qn^T is an orthogonal projector into the tangent space.
2603    Since Q is symmetric, this is equivalent to multipyling by Q and taking the last two entries.
2604    In total, we have a system of 3 equations in 3 unknowns:
2605 
2606      f(y) = 0                       1 equation
2607      Qn^T (y - x) = 0               2 equations
2608 
2609    Here, we compute the residual and Jacobian of this system.
2610 */
2611 static void TPSNearestPointResJac(TPSEvaluateFunc feval, const PetscScalar x[], const PetscScalar y[], PetscScalar res[], PetscScalar J[])
2612 {
2613   PetscReal yreal[3] = {PetscRealPart(y[0]), PetscRealPart(y[1]), PetscRealPart(y[2])};
2614   PetscReal d[3]     = {PetscRealPart(y[0] - x[0]), PetscRealPart(y[1] - x[1]), PetscRealPart(y[2] - x[2])};
2615   PetscReal f, grad[3], n[3], norm, norm_y[3], nd, nd_y[3], sign;
2616   PetscReal n_y[3][3] = {
2617     {0, 0, 0},
2618     {0, 0, 0},
2619     {0, 0, 0}
2620   };
2621 
2622   feval(yreal, &f, grad, n_y);
2623 
2624   for (PetscInt i = 0; i < 3; i++) n[i] = grad[i];
2625   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2626   for (PetscInt i = 0; i < 3; i++) norm_y[i] = 1. / norm * n[i] * n_y[i][i];
2627 
2628   // Define the Householder reflector
2629   sign = n[0] >= 0 ? 1. : -1.;
2630   n[0] += norm * sign;
2631   for (PetscInt i = 0; i < 3; i++) n_y[0][i] += norm_y[i] * sign;
2632 
2633   norm      = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2634   norm_y[0] = 1. / norm * (n[0] * n_y[0][0]);
2635   norm_y[1] = 1. / norm * (n[0] * n_y[0][1] + n[1] * n_y[1][1]);
2636   norm_y[2] = 1. / norm * (n[0] * n_y[0][2] + n[2] * n_y[2][2]);
2637 
2638   for (PetscInt i = 0; i < 3; i++) {
2639     n[i] /= norm;
2640     for (PetscInt j = 0; j < 3; j++) {
2641       // note that n[i] is n_old[i]/norm when executing the code below
2642       n_y[i][j] = n_y[i][j] / norm - n[i] / norm * norm_y[j];
2643     }
2644   }
2645 
2646   nd = n[0] * d[0] + n[1] * d[1] + n[2] * d[2];
2647   for (PetscInt i = 0; i < 3; i++) nd_y[i] = n[i] + n_y[0][i] * d[0] + n_y[1][i] * d[1] + n_y[2][i] * d[2];
2648 
2649   res[0] = f;
2650   res[1] = d[1] - 2 * n[1] * nd;
2651   res[2] = d[2] - 2 * n[2] * nd;
2652   // J[j][i] is J_{ij} (column major)
2653   for (PetscInt j = 0; j < 3; j++) {
2654     J[0 + j * 3] = grad[j];
2655     J[1 + j * 3] = (j == 1) * 1. - 2 * (n_y[1][j] * nd + n[1] * nd_y[j]);
2656     J[2 + j * 3] = (j == 2) * 1. - 2 * (n_y[2][j] * nd + n[2] * nd_y[j]);
2657   }
2658 }
2659 
2660 /*
2661    Project x to the nearest point on the implicit surface using Newton's method.
2662 */
2663 static PetscErrorCode TPSNearestPoint(TPSEvaluateFunc feval, PetscScalar x[])
2664 {
2665   PetscScalar y[3] = {x[0], x[1], x[2]}; // Initial guess
2666 
2667   PetscFunctionBegin;
2668   for (PetscInt iter = 0; iter < 10; iter++) {
2669     PetscScalar res[3], J[9];
2670     PetscReal   resnorm;
2671     TPSNearestPointResJac(feval, x, y, res, J);
2672     resnorm = PetscSqrtReal(PetscSqr(PetscRealPart(res[0])) + PetscSqr(PetscRealPart(res[1])) + PetscSqr(PetscRealPart(res[2])));
2673     if (0) { // Turn on this monitor if you need to confirm quadratic convergence
2674       PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%" PetscInt_FMT "] res [%g %g %g]\n", iter, (double)PetscRealPart(res[0]), (double)PetscRealPart(res[1]), (double)PetscRealPart(res[2])));
2675     }
2676     if (resnorm < PETSC_SMALL) break;
2677 
2678     // Take the Newton step
2679     PetscCall(PetscKernel_A_gets_inverse_A_3(J, 0., PETSC_FALSE, NULL));
2680     PetscKernel_v_gets_v_minus_A_times_w_3(y, J, res);
2681   }
2682   for (PetscInt i = 0; i < 3; i++) x[i] = y[i];
2683   PetscFunctionReturn(0);
2684 }
2685 
2686 const char *const DMPlexTPSTypes[] = {"SCHWARZ_P", "GYROID", "DMPlexTPSType", "DMPLEX_TPS_", NULL};
2687 
2688 static PetscErrorCode DMPlexCreateTPSMesh_Internal(DM dm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness)
2689 {
2690   PetscMPIInt rank;
2691   PetscInt    topoDim = 2, spaceDim = 3, numFaces = 0, numVertices = 0, numEdges = 0;
2692   PetscInt(*edges)[2] = NULL, *edgeSets = NULL;
2693   PetscInt            *cells_flat = NULL;
2694   PetscReal           *vtxCoords  = NULL;
2695   TPSEvaluateFunc      evalFunc   = NULL;
2696   PetscSimplePointFunc normalFunc = NULL;
2697   DMLabel              label;
2698 
2699   PetscFunctionBegin;
2700   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2701   PetscCheck((layers != 0) ^ (thickness == 0.), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Layers %" PetscInt_FMT " must be nonzero iff thickness %g is nonzero", layers, (double)thickness);
2702   switch (tpstype) {
2703   case DMPLEX_TPS_SCHWARZ_P:
2704     PetscCheck(!periodic || (periodic[0] == DM_BOUNDARY_NONE && periodic[1] == DM_BOUNDARY_NONE && periodic[2] == DM_BOUNDARY_NONE), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Schwarz P does not support periodic meshes");
2705     if (rank == 0) {
2706       PetscInt(*cells)[6][4][4] = NULL; // [junction, junction-face, cell, conn]
2707       PetscInt  Njunctions = 0, Ncuts = 0, Npipes[3], vcount;
2708       PetscReal L = 1;
2709 
2710       Npipes[0]   = (extent[0] + 1) * extent[1] * extent[2];
2711       Npipes[1]   = extent[0] * (extent[1] + 1) * extent[2];
2712       Npipes[2]   = extent[0] * extent[1] * (extent[2] + 1);
2713       Njunctions  = extent[0] * extent[1] * extent[2];
2714       Ncuts       = 2 * (extent[0] * extent[1] + extent[1] * extent[2] + extent[2] * extent[0]);
2715       numVertices = 4 * (Npipes[0] + Npipes[1] + Npipes[2]) + 8 * Njunctions;
2716       PetscCall(PetscMalloc1(3 * numVertices, &vtxCoords));
2717       PetscCall(PetscMalloc1(Njunctions, &cells));
2718       PetscCall(PetscMalloc1(Ncuts * 4, &edges));
2719       PetscCall(PetscMalloc1(Ncuts * 4, &edgeSets));
2720       // x-normal pipes
2721       vcount = 0;
2722       for (PetscInt i = 0; i < extent[0] + 1; i++) {
2723         for (PetscInt j = 0; j < extent[1]; j++) {
2724           for (PetscInt k = 0; k < extent[2]; k++) {
2725             for (PetscInt l = 0; l < 4; l++) {
2726               vtxCoords[vcount++] = (2 * i - 1) * L;
2727               vtxCoords[vcount++] = 2 * j * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2728               vtxCoords[vcount++] = 2 * k * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2729             }
2730           }
2731         }
2732       }
2733       // y-normal pipes
2734       for (PetscInt i = 0; i < extent[0]; i++) {
2735         for (PetscInt j = 0; j < extent[1] + 1; j++) {
2736           for (PetscInt k = 0; k < extent[2]; k++) {
2737             for (PetscInt l = 0; l < 4; l++) {
2738               vtxCoords[vcount++] = 2 * i * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2739               vtxCoords[vcount++] = (2 * j - 1) * L;
2740               vtxCoords[vcount++] = 2 * k * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2741             }
2742           }
2743         }
2744       }
2745       // z-normal pipes
2746       for (PetscInt i = 0; i < extent[0]; i++) {
2747         for (PetscInt j = 0; j < extent[1]; j++) {
2748           for (PetscInt k = 0; k < extent[2] + 1; k++) {
2749             for (PetscInt l = 0; l < 4; l++) {
2750               vtxCoords[vcount++] = 2 * i * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2751               vtxCoords[vcount++] = 2 * j * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2752               vtxCoords[vcount++] = (2 * k - 1) * L;
2753             }
2754           }
2755         }
2756       }
2757       // junctions
2758       for (PetscInt i = 0; i < extent[0]; i++) {
2759         for (PetscInt j = 0; j < extent[1]; j++) {
2760           for (PetscInt k = 0; k < extent[2]; k++) {
2761             const PetscInt J = (i * extent[1] + j) * extent[2] + k, Jvoff = (Npipes[0] + Npipes[1] + Npipes[2]) * 4 + J * 8;
2762             PetscCheck(vcount / 3 == Jvoff, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected vertex count");
2763             for (PetscInt ii = 0; ii < 2; ii++) {
2764               for (PetscInt jj = 0; jj < 2; jj++) {
2765                 for (PetscInt kk = 0; kk < 2; kk++) {
2766                   double Ls           = (1 - sqrt(2) / 4) * L;
2767                   vtxCoords[vcount++] = 2 * i * L + (2 * ii - 1) * Ls;
2768                   vtxCoords[vcount++] = 2 * j * L + (2 * jj - 1) * Ls;
2769                   vtxCoords[vcount++] = 2 * k * L + (2 * kk - 1) * Ls;
2770                 }
2771               }
2772             }
2773             const PetscInt jfaces[3][2][4] = {
2774               {{3, 1, 0, 2}, {7, 5, 4, 6}}, // x-aligned
2775               {{5, 4, 0, 1}, {7, 6, 2, 3}}, // y-aligned
2776               {{6, 2, 0, 4}, {7, 3, 1, 5}}  // z-aligned
2777             };
2778             const PetscInt pipe_lo[3] = {// vertex numbers of pipes
2779                                          ((i * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + Npipes[0] + Npipes[1]) * 4};
2780             const PetscInt pipe_hi[3] = {// vertex numbers of pipes
2781                                          (((i + 1) * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j + 1) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + 1 + Npipes[0] + Npipes[1]) * 4};
2782             for (PetscInt dir = 0; dir < 3; dir++) { // x,y,z
2783               const PetscInt ijk[3] = {i, j, k};
2784               for (PetscInt l = 0; l < 4; l++) { // rotations
2785                 cells[J][dir * 2 + 0][l][0] = pipe_lo[dir] + l;
2786                 cells[J][dir * 2 + 0][l][1] = Jvoff + jfaces[dir][0][l];
2787                 cells[J][dir * 2 + 0][l][2] = Jvoff + jfaces[dir][0][(l - 1 + 4) % 4];
2788                 cells[J][dir * 2 + 0][l][3] = pipe_lo[dir] + (l - 1 + 4) % 4;
2789                 cells[J][dir * 2 + 1][l][0] = Jvoff + jfaces[dir][1][l];
2790                 cells[J][dir * 2 + 1][l][1] = pipe_hi[dir] + l;
2791                 cells[J][dir * 2 + 1][l][2] = pipe_hi[dir] + (l - 1 + 4) % 4;
2792                 cells[J][dir * 2 + 1][l][3] = Jvoff + jfaces[dir][1][(l - 1 + 4) % 4];
2793                 if (ijk[dir] == 0) {
2794                   edges[numEdges][0] = pipe_lo[dir] + l;
2795                   edges[numEdges][1] = pipe_lo[dir] + (l + 1) % 4;
2796                   edgeSets[numEdges] = dir * 2 + 1;
2797                   numEdges++;
2798                 }
2799                 if (ijk[dir] + 1 == extent[dir]) {
2800                   edges[numEdges][0] = pipe_hi[dir] + l;
2801                   edges[numEdges][1] = pipe_hi[dir] + (l + 1) % 4;
2802                   edgeSets[numEdges] = dir * 2 + 2;
2803                   numEdges++;
2804                 }
2805               }
2806             }
2807           }
2808         }
2809       }
2810       PetscCheck(numEdges == Ncuts * 4, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Edge count %" PetscInt_FMT " incompatible with number of cuts %" PetscInt_FMT, numEdges, Ncuts);
2811       numFaces   = 24 * Njunctions;
2812       cells_flat = cells[0][0][0];
2813     }
2814     evalFunc   = TPSEvaluate_SchwarzP;
2815     normalFunc = TPSExtrudeNormalFunc_SchwarzP;
2816     break;
2817   case DMPLEX_TPS_GYROID:
2818     if (rank == 0) {
2819       // This is a coarse mesh approximation of the gyroid shifted to being the zero of the level set
2820       //
2821       //     sin(pi*x)*cos(pi*(y+1/2)) + sin(pi*(y+1/2))*cos(pi*(z+1/4)) + sin(pi*(z+1/4))*cos(x)
2822       //
2823       // on the cell [0,2]^3.
2824       //
2825       // Think about dividing that cell into four columns, and focus on the column [0,1]x[0,1]x[0,2].
2826       // If you looked at the gyroid in that column at different slices of z you would see that it kind of spins
2827       // like a boomerang:
2828       //
2829       //     z = 0          z = 1/4        z = 1/2        z = 3/4     //
2830       //     -----          -------        -------        -------     //
2831       //                                                              //
2832       //     +       +      +       +      +       +      +   \   +   //
2833       //      \                                   /            \      //
2834       //       \            `-_   _-'            /              }     //
2835       //        *-_            `-'            _-'              /      //
2836       //     +     `-+      +       +      +-'     +      +   /   +   //
2837       //                                                              //
2838       //                                                              //
2839       //     z = 1          z = 5/4        z = 3/2        z = 7/4     //
2840       //     -----          -------        -------        -------     //
2841       //                                                              //
2842       //     +-_     +      +       +      +     _-+      +   /   +   //
2843       //        `-_            _-_            _-`            /        //
2844       //           \        _-'   `-_        /              {         //
2845       //            \                       /                \        //
2846       //     +       +      +       +      +       +      +   \   +   //
2847       //
2848       //
2849       // This course mesh approximates each of these slices by two line segments,
2850       // and then connects the segments in consecutive layers with quadrilateral faces.
2851       // All of the end points of the segments are multiples of 1/4 except for the
2852       // point * in the picture for z = 0 above and the similar points in other layers.
2853       // That point is at (gamma, gamma, 0), where gamma is calculated below.
2854       //
2855       // The column  [1,2]x[1,2]x[0,2] looks the same as this column;
2856       // The columns [1,2]x[0,1]x[0,2] and [0,1]x[1,2]x[0,2] are mirror images.
2857       //
2858       // As for how this method turned into the names given to the vertices:
2859       // that was not systematic, it was just the way it worked out in my handwritten notes.
2860 
2861       PetscInt facesPerBlock = 64;
2862       PetscInt vertsPerBlock = 56;
2863       PetscInt extentPlus[3];
2864       PetscInt numBlocks, numBlocksPlus;
2865       const PetscInt A = 0, B = 1, C = 2, D = 3, E = 4, F = 5, G = 6, H = 7, II = 8, J = 9, K = 10, L = 11, M = 12, N = 13, O = 14, P = 15, Q = 16, R = 17, S = 18, T = 19, U = 20, V = 21, W = 22, X = 23, Y = 24, Z = 25, Ap = 26, Bp = 27, Cp = 28, Dp = 29, Ep = 30, Fp = 31, Gp = 32, Hp = 33, Ip = 34, Jp = 35, Kp = 36, Lp = 37, Mp = 38, Np = 39, Op = 40, Pp = 41, Qp = 42, Rp = 43, Sp = 44, Tp = 45, Up = 46, Vp = 47, Wp = 48, Xp = 49, Yp = 50, Zp = 51, Aq = 52, Bq = 53, Cq = 54, Dq = 55;
2866       const PetscInt pattern[64][4] = {
2867   /* face to vertex within the coarse discretization of a single gyroid block */
2868   /* layer 0 */
2869         {A,           C,           K,           G          },
2870         {C,           B,           II,          K          },
2871         {D,           A,           H,           L          },
2872         {B + 56 * 1,  D,           L,           J          },
2873         {E,           B + 56 * 1,  J,           N          },
2874         {A + 56 * 2,  E,           N,           H + 56 * 2 },
2875         {F,           A + 56 * 2,  G + 56 * 2,  M          },
2876         {B,           F,           M,           II         },
2877  /* layer 1 */
2878         {G,           K,           Q,           O          },
2879         {K,           II,          P,           Q          },
2880         {L,           H,           O + 56 * 1,  R          },
2881         {J,           L,           R,           P          },
2882         {N,           J,           P,           S          },
2883         {H + 56 * 2,  N,           S,           O + 56 * 3 },
2884         {M,           G + 56 * 2,  O + 56 * 2,  T          },
2885         {II,          M,           T,           P          },
2886  /* layer 2 */
2887         {O,           Q,           Y,           U          },
2888         {Q,           P,           W,           Y          },
2889         {R,           O + 56 * 1,  U + 56 * 1,  Ap         },
2890         {P,           R,           Ap,          W          },
2891         {S,           P,           X,           Bp         },
2892         {O + 56 * 3,  S,           Bp,          V + 56 * 1 },
2893         {T,           O + 56 * 2,  V,           Z          },
2894         {P,           T,           Z,           X          },
2895  /* layer 3 */
2896         {U,           Y,           Ep,          Dp         },
2897         {Y,           W,           Cp,          Ep         },
2898         {Ap,          U + 56 * 1,  Dp + 56 * 1, Gp         },
2899         {W,           Ap,          Gp,          Cp         },
2900         {Bp,          X,           Cp + 56 * 2, Fp         },
2901         {V + 56 * 1,  Bp,          Fp,          Dp + 56 * 1},
2902         {Z,           V,           Dp,          Hp         },
2903         {X,           Z,           Hp,          Cp + 56 * 2},
2904  /* layer 4 */
2905         {Dp,          Ep,          Mp,          Kp         },
2906         {Ep,          Cp,          Ip,          Mp         },
2907         {Gp,          Dp + 56 * 1, Lp,          Np         },
2908         {Cp,          Gp,          Np,          Jp         },
2909         {Fp,          Cp + 56 * 2, Jp + 56 * 2, Pp         },
2910         {Dp + 56 * 1, Fp,          Pp,          Lp         },
2911         {Hp,          Dp,          Kp,          Op         },
2912         {Cp + 56 * 2, Hp,          Op,          Ip + 56 * 2},
2913  /* layer 5 */
2914         {Kp,          Mp,          Sp,          Rp         },
2915         {Mp,          Ip,          Qp,          Sp         },
2916         {Np,          Lp,          Rp,          Tp         },
2917         {Jp,          Np,          Tp,          Qp + 56 * 1},
2918         {Pp,          Jp + 56 * 2, Qp + 56 * 3, Up         },
2919         {Lp,          Pp,          Up,          Rp         },
2920         {Op,          Kp,          Rp,          Vp         },
2921         {Ip + 56 * 2, Op,          Vp,          Qp + 56 * 2},
2922  /* layer 6 */
2923         {Rp,          Sp,          Aq,          Yp         },
2924         {Sp,          Qp,          Wp,          Aq         },
2925         {Tp,          Rp,          Yp,          Cq         },
2926         {Qp + 56 * 1, Tp,          Cq,          Wp + 56 * 1},
2927         {Up,          Qp + 56 * 3, Xp + 56 * 1, Dq         },
2928         {Rp,          Up,          Dq,          Zp         },
2929         {Vp,          Rp,          Zp,          Bq         },
2930         {Qp + 56 * 2, Vp,          Bq,          Xp         },
2931  /* layer 7 (the top is the periodic image of the bottom of layer 0) */
2932         {Yp,          Aq,          C + 56 * 4,  A + 56 * 4 },
2933         {Aq,          Wp,          B + 56 * 4,  C + 56 * 4 },
2934         {Cq,          Yp,          A + 56 * 4,  D + 56 * 4 },
2935         {Wp + 56 * 1, Cq,          D + 56 * 4,  B + 56 * 5 },
2936         {Dq,          Xp + 56 * 1, B + 56 * 5,  E + 56 * 4 },
2937         {Zp,          Dq,          E + 56 * 4,  A + 56 * 6 },
2938         {Bq,          Zp,          A + 56 * 6,  F + 56 * 4 },
2939         {Xp,          Bq,          F + 56 * 4,  B + 56 * 4 }
2940       };
2941       const PetscReal gamma                = PetscAcosReal((PetscSqrtReal(3.) - 1.) / PetscSqrtReal(2.)) / PETSC_PI;
2942       const PetscReal patternCoords[56][3] = {
2943         {1.,        0.,        0.  }, /* A  */
2944         {0.,        1.,        0.  }, /* B  */
2945         {gamma,     gamma,     0.  }, /* C  */
2946         {1 + gamma, 1 - gamma, 0.  }, /* D  */
2947         {2 - gamma, 2 - gamma, 0.  }, /* E  */
2948         {1 - gamma, 1 + gamma, 0.  }, /* F  */
2949 
2950         {.5,        0,         .25 }, /* G  */
2951         {1.5,       0.,        .25 }, /* H  */
2952         {.5,        1.,        .25 }, /* II */
2953         {1.5,       1.,        .25 }, /* J  */
2954         {.25,       .5,        .25 }, /* K  */
2955         {1.25,      .5,        .25 }, /* L  */
2956         {.75,       1.5,       .25 }, /* M  */
2957         {1.75,      1.5,       .25 }, /* N  */
2958 
2959         {0.,        0.,        .5  }, /* O  */
2960         {1.,        1.,        .5  }, /* P  */
2961         {gamma,     1 - gamma, .5  }, /* Q  */
2962         {1 + gamma, gamma,     .5  }, /* R  */
2963         {2 - gamma, 1 + gamma, .5  }, /* S  */
2964         {1 - gamma, 2 - gamma, .5  }, /* T  */
2965 
2966         {0.,        .5,        .75 }, /* U  */
2967         {0.,        1.5,       .75 }, /* V  */
2968         {1.,        .5,        .75 }, /* W  */
2969         {1.,        1.5,       .75 }, /* X  */
2970         {.5,        .75,       .75 }, /* Y  */
2971         {.5,        1.75,      .75 }, /* Z  */
2972         {1.5,       .25,       .75 }, /* Ap */
2973         {1.5,       1.25,      .75 }, /* Bp */
2974 
2975         {1.,        0.,        1.  }, /* Cp */
2976         {0.,        1.,        1.  }, /* Dp */
2977         {1 - gamma, 1 - gamma, 1.  }, /* Ep */
2978         {1 + gamma, 1 + gamma, 1.  }, /* Fp */
2979         {2 - gamma, gamma,     1.  }, /* Gp */
2980         {gamma,     2 - gamma, 1.  }, /* Hp */
2981 
2982         {.5,        0.,        1.25}, /* Ip */
2983         {1.5,       0.,        1.25}, /* Jp */
2984         {.5,        1.,        1.25}, /* Kp */
2985         {1.5,       1.,        1.25}, /* Lp */
2986         {.75,       .5,        1.25}, /* Mp */
2987         {1.75,      .5,        1.25}, /* Np */
2988         {.25,       1.5,       1.25}, /* Op */
2989         {1.25,      1.5,       1.25}, /* Pp */
2990 
2991         {0.,        0.,        1.5 }, /* Qp */
2992         {1.,        1.,        1.5 }, /* Rp */
2993         {1 - gamma, gamma,     1.5 }, /* Sp */
2994         {2 - gamma, 1 - gamma, 1.5 }, /* Tp */
2995         {1 + gamma, 2 - gamma, 1.5 }, /* Up */
2996         {gamma,     1 + gamma, 1.5 }, /* Vp */
2997 
2998         {0.,        .5,        1.75}, /* Wp */
2999         {0.,        1.5,       1.75}, /* Xp */
3000         {1.,        .5,        1.75}, /* Yp */
3001         {1.,        1.5,       1.75}, /* Zp */
3002         {.5,        .25,       1.75}, /* Aq */
3003         {.5,        1.25,      1.75}, /* Bq */
3004         {1.5,       .75,       1.75}, /* Cq */
3005         {1.5,       1.75,      1.75}, /* Dq */
3006       };
3007       PetscInt(*cells)[64][4] = NULL;
3008       PetscBool *seen;
3009       PetscInt  *vertToTrueVert;
3010       PetscInt   count;
3011 
3012       for (PetscInt i = 0; i < 3; i++) extentPlus[i] = extent[i] + 1;
3013       numBlocks = 1;
3014       for (PetscInt i = 0; i < 3; i++) numBlocks *= extent[i];
3015       numBlocksPlus = 1;
3016       for (PetscInt i = 0; i < 3; i++) numBlocksPlus *= extentPlus[i];
3017       numFaces = numBlocks * facesPerBlock;
3018       PetscCall(PetscMalloc1(numBlocks, &cells));
3019       PetscCall(PetscCalloc1(numBlocksPlus * vertsPerBlock, &seen));
3020       for (PetscInt k = 0; k < extent[2]; k++) {
3021         for (PetscInt j = 0; j < extent[1]; j++) {
3022           for (PetscInt i = 0; i < extent[0]; i++) {
3023             for (PetscInt f = 0; f < facesPerBlock; f++) {
3024               for (PetscInt v = 0; v < 4; v++) {
3025                 PetscInt vertRaw     = pattern[f][v];
3026                 PetscInt blockidx    = vertRaw / 56;
3027                 PetscInt patternvert = vertRaw % 56;
3028                 PetscInt xplus       = (blockidx & 1);
3029                 PetscInt yplus       = (blockidx & 2) >> 1;
3030                 PetscInt zplus       = (blockidx & 4) >> 2;
3031                 PetscInt zcoord      = (periodic && periodic[2] == DM_BOUNDARY_PERIODIC) ? ((k + zplus) % extent[2]) : (k + zplus);
3032                 PetscInt ycoord      = (periodic && periodic[1] == DM_BOUNDARY_PERIODIC) ? ((j + yplus) % extent[1]) : (j + yplus);
3033                 PetscInt xcoord      = (periodic && periodic[0] == DM_BOUNDARY_PERIODIC) ? ((i + xplus) % extent[0]) : (i + xplus);
3034                 PetscInt vert        = ((zcoord * extentPlus[1] + ycoord) * extentPlus[0] + xcoord) * 56 + patternvert;
3035 
3036                 cells[(k * extent[1] + j) * extent[0] + i][f][v] = vert;
3037                 seen[vert]                                       = PETSC_TRUE;
3038               }
3039             }
3040           }
3041         }
3042       }
3043       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++)
3044         if (seen[i]) numVertices++;
3045       count = 0;
3046       PetscCall(PetscMalloc1(numBlocksPlus * vertsPerBlock, &vertToTrueVert));
3047       PetscCall(PetscMalloc1(numVertices * 3, &vtxCoords));
3048       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) vertToTrueVert[i] = -1;
3049       for (PetscInt k = 0; k < extentPlus[2]; k++) {
3050         for (PetscInt j = 0; j < extentPlus[1]; j++) {
3051           for (PetscInt i = 0; i < extentPlus[0]; i++) {
3052             for (PetscInt v = 0; v < vertsPerBlock; v++) {
3053               PetscInt vIdx = ((k * extentPlus[1] + j) * extentPlus[0] + i) * vertsPerBlock + v;
3054 
3055               if (seen[vIdx]) {
3056                 PetscInt thisVert;
3057 
3058                 vertToTrueVert[vIdx] = thisVert = count++;
3059 
3060                 for (PetscInt d = 0; d < 3; d++) vtxCoords[3 * thisVert + d] = patternCoords[v][d];
3061                 vtxCoords[3 * thisVert + 0] += i * 2;
3062                 vtxCoords[3 * thisVert + 1] += j * 2;
3063                 vtxCoords[3 * thisVert + 2] += k * 2;
3064               }
3065             }
3066           }
3067         }
3068       }
3069       for (PetscInt i = 0; i < numBlocks; i++) {
3070         for (PetscInt f = 0; f < facesPerBlock; f++) {
3071           for (PetscInt v = 0; v < 4; v++) cells[i][f][v] = vertToTrueVert[cells[i][f][v]];
3072         }
3073       }
3074       PetscCall(PetscFree(vertToTrueVert));
3075       PetscCall(PetscFree(seen));
3076       cells_flat = cells[0][0];
3077       numEdges   = 0;
3078       for (PetscInt i = 0; i < numFaces; i++) {
3079         for (PetscInt e = 0; e < 4; e++) {
3080           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3081           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3082 
3083           for (PetscInt d = 0; d < 3; d++) {
3084             if (!periodic || periodic[0] != DM_BOUNDARY_PERIODIC) {
3085               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) numEdges++;
3086               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) numEdges++;
3087             }
3088           }
3089         }
3090       }
3091       PetscCall(PetscMalloc1(numEdges, &edges));
3092       PetscCall(PetscMalloc1(numEdges, &edgeSets));
3093       for (PetscInt edge = 0, i = 0; i < numFaces; i++) {
3094         for (PetscInt e = 0; e < 4; e++) {
3095           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3096           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3097 
3098           for (PetscInt d = 0; d < 3; d++) {
3099             if (!periodic || periodic[d] != DM_BOUNDARY_PERIODIC) {
3100               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) {
3101                 edges[edge][0]   = ev[0];
3102                 edges[edge][1]   = ev[1];
3103                 edgeSets[edge++] = 2 * d;
3104               }
3105               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) {
3106                 edges[edge][0]   = ev[0];
3107                 edges[edge][1]   = ev[1];
3108                 edgeSets[edge++] = 2 * d + 1;
3109               }
3110             }
3111           }
3112         }
3113       }
3114     }
3115     evalFunc   = TPSEvaluate_Gyroid;
3116     normalFunc = TPSExtrudeNormalFunc_Gyroid;
3117     break;
3118   }
3119 
3120   PetscCall(DMSetDimension(dm, topoDim));
3121   if (rank == 0) PetscCall(DMPlexBuildFromCellList(dm, numFaces, numVertices, 4, cells_flat));
3122   else PetscCall(DMPlexBuildFromCellList(dm, 0, 0, 0, NULL));
3123   PetscCall(PetscFree(cells_flat));
3124   {
3125     DM idm;
3126     PetscCall(DMPlexInterpolate(dm, &idm));
3127     PetscCall(DMPlexReplace_Internal(dm, &idm));
3128   }
3129   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, vtxCoords));
3130   else PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, NULL));
3131   PetscCall(PetscFree(vtxCoords));
3132 
3133   PetscCall(DMCreateLabel(dm, "Face Sets"));
3134   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3135   for (PetscInt e = 0; e < numEdges; e++) {
3136     PetscInt        njoin;
3137     const PetscInt *join, verts[] = {numFaces + edges[e][0], numFaces + edges[e][1]};
3138     PetscCall(DMPlexGetJoin(dm, 2, verts, &njoin, &join));
3139     PetscCheck(njoin == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected unique join of vertices %" PetscInt_FMT " and %" PetscInt_FMT, edges[e][0], edges[e][1]);
3140     PetscCall(DMLabelSetValue(label, join[0], edgeSets[e]));
3141     PetscCall(DMPlexRestoreJoin(dm, 2, verts, &njoin, &join));
3142   }
3143   PetscCall(PetscFree(edges));
3144   PetscCall(PetscFree(edgeSets));
3145   if (tps_distribute) {
3146     DM               pdm = NULL;
3147     PetscPartitioner part;
3148 
3149     PetscCall(DMPlexGetPartitioner(dm, &part));
3150     PetscCall(PetscPartitionerSetFromOptions(part));
3151     PetscCall(DMPlexDistribute(dm, 0, NULL, &pdm));
3152     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
3153     // Do not auto-distribute again
3154     PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
3155   }
3156 
3157   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3158   for (PetscInt refine = 0; refine < refinements; refine++) {
3159     PetscInt     m;
3160     DM           dmf;
3161     Vec          X;
3162     PetscScalar *x;
3163     PetscCall(DMRefine(dm, MPI_COMM_NULL, &dmf));
3164     PetscCall(DMPlexReplace_Internal(dm, &dmf));
3165 
3166     PetscCall(DMGetCoordinatesLocal(dm, &X));
3167     PetscCall(VecGetLocalSize(X, &m));
3168     PetscCall(VecGetArray(X, &x));
3169     for (PetscInt i = 0; i < m; i += 3) PetscCall(TPSNearestPoint(evalFunc, &x[i]));
3170     PetscCall(VecRestoreArray(X, &x));
3171   }
3172 
3173   // Face Sets has already been propagated to new vertices during refinement; this propagates to the initial vertices.
3174   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3175   PetscCall(DMPlexLabelComplete(dm, label));
3176 
3177   if (thickness > 0) {
3178     DM              edm, cdm, ecdm;
3179     DMPlexTransform tr;
3180     const char     *prefix;
3181     PetscOptions    options;
3182     // Code from DMPlexExtrude
3183     PetscCall(DMPlexTransformCreate(PetscObjectComm((PetscObject)dm), &tr));
3184     PetscCall(DMPlexTransformSetDM(tr, dm));
3185     PetscCall(DMPlexTransformSetType(tr, DMPLEXEXTRUDE));
3186     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
3187     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)tr, prefix));
3188     PetscCall(PetscObjectGetOptions((PetscObject)dm, &options));
3189     PetscCall(PetscObjectSetOptions((PetscObject)tr, options));
3190     PetscCall(DMPlexTransformExtrudeSetLayers(tr, layers));
3191     PetscCall(DMPlexTransformExtrudeSetThickness(tr, thickness));
3192     PetscCall(DMPlexTransformExtrudeSetTensor(tr, PETSC_FALSE));
3193     PetscCall(DMPlexTransformExtrudeSetSymmetric(tr, PETSC_TRUE));
3194     PetscCall(DMPlexTransformExtrudeSetNormalFunction(tr, normalFunc));
3195     PetscCall(DMPlexTransformSetFromOptions(tr));
3196     PetscCall(PetscObjectSetOptions((PetscObject)tr, NULL));
3197     PetscCall(DMPlexTransformSetUp(tr));
3198     PetscCall(PetscObjectViewFromOptions((PetscObject)tr, NULL, "-dm_plex_tps_transform_view"));
3199     PetscCall(DMPlexTransformApply(tr, dm, &edm));
3200     PetscCall(DMCopyDisc(dm, edm));
3201     PetscCall(DMGetCoordinateDM(dm, &cdm));
3202     PetscCall(DMGetCoordinateDM(edm, &ecdm));
3203     PetscCall(DMCopyDisc(cdm, ecdm));
3204     PetscCall(DMPlexTransformCreateDiscLabels(tr, edm));
3205     PetscCall(DMPlexTransformDestroy(&tr));
3206     if (edm) {
3207       ((DM_Plex *)edm->data)->printFEM    = ((DM_Plex *)dm->data)->printFEM;
3208       ((DM_Plex *)edm->data)->printL2     = ((DM_Plex *)dm->data)->printL2;
3209       ((DM_Plex *)edm->data)->printLocate = ((DM_Plex *)dm->data)->printLocate;
3210     }
3211     PetscCall(DMPlexReplace_Internal(dm, &edm));
3212   }
3213   PetscFunctionReturn(0);
3214 }
3215 
3216 /*@
3217   DMPlexCreateTPSMesh - Create a distributed, interpolated mesh of a triply-periodic surface
3218 
3219   Collective
3220 
3221   Input Parameters:
3222 + comm   - The communicator for the DM object
3223 . tpstype - Type of triply-periodic surface
3224 . extent - Array of length 3 containing number of periods in each direction
3225 . periodic - array of length 3 with periodicity, or NULL for non-periodic
3226 . tps_distribute - Distribute 2D manifold mesh prior to refinement and extrusion (more scalable)
3227 . refinements - Number of factor-of-2 refinements of 2D manifold mesh
3228 . layers - Number of cell layers extruded in normal direction
3229 - thickness - Thickness in normal direction
3230 
3231   Output Parameter:
3232 . dm  - The DM object
3233 
3234   Notes:
3235   This meshes the surface of the Schwarz P or Gyroid surfaces.  Schwarz P is is the simplest member of the triply-periodic minimal surfaces.
3236   https://en.wikipedia.org/wiki/Schwarz_minimal_surface#Schwarz_P_(%22Primitive%22) and can be cut with "clean" boundaries.
3237   The Gyroid (https://en.wikipedia.org/wiki/Gyroid) is another triply-periodic minimal surface with applications in additive manufacturing; it is much more difficult to "cut" since there are no planes of symmetry.
3238   Our implementation creates a very coarse mesh of the surface and refines (by 4-way splitting) as many times as requested.
3239   On each refinement, all vertices are projected to their nearest point on the surface.
3240   This projection could readily be extended to related surfaces.
3241 
3242   The face (edge) sets for the Schwarz P surface are numbered 1(-x), 2(+x), 3(-y), 4(+y), 5(-z), 6(+z).
3243   When the mesh is refined, "Face Sets" contain the new vertices (created during refinement).  Use DMPlexLabelComplete() to propagate to coarse-level vertices.
3244 
3245   References:
3246 . * - Maskery et al, Insights into the mechanical properties of several triply periodic minimal surface lattice structures made by polymer additive manufacturing, 2017. https://doi.org/10.1016/j.polymer.2017.11.049
3247 
3248   Developer Notes:
3249   The Gyroid mesh does not currently mark boundary sets.
3250 
3251   Level: beginner
3252 
3253 .seealso: `DMPlexCreateSphereMesh()`, `DMSetType()`, `DMCreate()`
3254 @*/
3255 PetscErrorCode DMPlexCreateTPSMesh(MPI_Comm comm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness, DM *dm)
3256 {
3257   PetscFunctionBegin;
3258   PetscCall(DMCreate(comm, dm));
3259   PetscCall(DMSetType(*dm, DMPLEX));
3260   PetscCall(DMPlexCreateTPSMesh_Internal(*dm, tpstype, extent, periodic, tps_distribute, refinements, layers, thickness));
3261   PetscFunctionReturn(0);
3262 }
3263 
3264 /*@
3265   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
3266 
3267   Collective
3268 
3269   Input Parameters:
3270 + comm    - The communicator for the DM object
3271 . dim     - The dimension
3272 . simplex - Use simplices, or tensor product cells
3273 - R       - The radius
3274 
3275   Output Parameter:
3276 . dm  - The DM object
3277 
3278   Level: beginner
3279 
3280 .seealso: `DMPlexCreateBallMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3281 @*/
3282 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscReal R, DM *dm)
3283 {
3284   PetscFunctionBegin;
3285   PetscValidPointer(dm, 5);
3286   PetscCall(DMCreate(comm, dm));
3287   PetscCall(DMSetType(*dm, DMPLEX));
3288   PetscCall(DMPlexCreateSphereMesh_Internal(*dm, dim, simplex, R));
3289   PetscFunctionReturn(0);
3290 }
3291 
3292 static PetscErrorCode DMPlexCreateBallMesh_Internal(DM dm, PetscInt dim, PetscReal R)
3293 {
3294   DM      sdm, vol;
3295   DMLabel bdlabel;
3296 
3297   PetscFunctionBegin;
3298   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
3299   PetscCall(DMSetType(sdm, DMPLEX));
3300   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sdm, "bd_"));
3301   PetscCall(DMPlexCreateSphereMesh_Internal(sdm, dim - 1, PETSC_TRUE, R));
3302   PetscCall(DMSetFromOptions(sdm));
3303   PetscCall(DMViewFromOptions(sdm, NULL, "-dm_view"));
3304   PetscCall(DMPlexGenerate(sdm, NULL, PETSC_TRUE, &vol));
3305   PetscCall(DMDestroy(&sdm));
3306   PetscCall(DMPlexReplace_Internal(dm, &vol));
3307   PetscCall(DMCreateLabel(dm, "marker"));
3308   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
3309   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
3310   PetscCall(DMPlexLabelComplete(dm, bdlabel));
3311   PetscFunctionReturn(0);
3312 }
3313 
3314 /*@
3315   DMPlexCreateBallMesh - Creates a simplex mesh on the d-dimensional ball, B^d.
3316 
3317   Collective
3318 
3319   Input Parameters:
3320 + comm  - The communicator for the DM object
3321 . dim   - The dimension
3322 - R     - The radius
3323 
3324   Output Parameter:
3325 . dm  - The DM object
3326 
3327   Options Database Keys:
3328 - bd_dm_refine - This will refine the surface mesh preserving the sphere geometry
3329 
3330   Level: beginner
3331 
3332 .seealso: `DMPlexCreateSphereMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3333 @*/
3334 PetscErrorCode DMPlexCreateBallMesh(MPI_Comm comm, PetscInt dim, PetscReal R, DM *dm)
3335 {
3336   PetscFunctionBegin;
3337   PetscCall(DMCreate(comm, dm));
3338   PetscCall(DMSetType(*dm, DMPLEX));
3339   PetscCall(DMPlexCreateBallMesh_Internal(*dm, dim, R));
3340   PetscFunctionReturn(0);
3341 }
3342 
3343 static PetscErrorCode DMPlexCreateReferenceCell_Internal(DM rdm, DMPolytopeType ct)
3344 {
3345   PetscFunctionBegin;
3346   switch (ct) {
3347   case DM_POLYTOPE_POINT: {
3348     PetscInt    numPoints[1]        = {1};
3349     PetscInt    coneSize[1]         = {0};
3350     PetscInt    cones[1]            = {0};
3351     PetscInt    coneOrientations[1] = {0};
3352     PetscScalar vertexCoords[1]     = {0.0};
3353 
3354     PetscCall(DMSetDimension(rdm, 0));
3355     PetscCall(DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3356   } break;
3357   case DM_POLYTOPE_SEGMENT: {
3358     PetscInt    numPoints[2]        = {2, 1};
3359     PetscInt    coneSize[3]         = {2, 0, 0};
3360     PetscInt    cones[2]            = {1, 2};
3361     PetscInt    coneOrientations[2] = {0, 0};
3362     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3363 
3364     PetscCall(DMSetDimension(rdm, 1));
3365     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3366   } break;
3367   case DM_POLYTOPE_POINT_PRISM_TENSOR: {
3368     PetscInt    numPoints[2]        = {2, 1};
3369     PetscInt    coneSize[3]         = {2, 0, 0};
3370     PetscInt    cones[2]            = {1, 2};
3371     PetscInt    coneOrientations[2] = {0, 0};
3372     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3373 
3374     PetscCall(DMSetDimension(rdm, 1));
3375     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3376   } break;
3377   case DM_POLYTOPE_TRIANGLE: {
3378     PetscInt    numPoints[2]        = {3, 1};
3379     PetscInt    coneSize[4]         = {3, 0, 0, 0};
3380     PetscInt    cones[3]            = {1, 2, 3};
3381     PetscInt    coneOrientations[3] = {0, 0, 0};
3382     PetscScalar vertexCoords[6]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0};
3383 
3384     PetscCall(DMSetDimension(rdm, 2));
3385     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3386   } break;
3387   case DM_POLYTOPE_QUADRILATERAL: {
3388     PetscInt    numPoints[2]        = {4, 1};
3389     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3390     PetscInt    cones[4]            = {1, 2, 3, 4};
3391     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3392     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0};
3393 
3394     PetscCall(DMSetDimension(rdm, 2));
3395     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3396   } break;
3397   case DM_POLYTOPE_SEG_PRISM_TENSOR: {
3398     PetscInt    numPoints[2]        = {4, 1};
3399     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3400     PetscInt    cones[4]            = {1, 2, 3, 4};
3401     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3402     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0};
3403 
3404     PetscCall(DMSetDimension(rdm, 2));
3405     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3406   } break;
3407   case DM_POLYTOPE_TETRAHEDRON: {
3408     PetscInt    numPoints[2]        = {4, 1};
3409     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3410     PetscInt    cones[4]            = {1, 2, 3, 4};
3411     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3412     PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0};
3413 
3414     PetscCall(DMSetDimension(rdm, 3));
3415     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3416   } break;
3417   case DM_POLYTOPE_HEXAHEDRON: {
3418     PetscInt    numPoints[2]        = {8, 1};
3419     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3420     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3421     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3422     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3423 
3424     PetscCall(DMSetDimension(rdm, 3));
3425     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3426   } break;
3427   case DM_POLYTOPE_TRI_PRISM: {
3428     PetscInt    numPoints[2]        = {6, 1};
3429     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3430     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3431     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3432     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3433 
3434     PetscCall(DMSetDimension(rdm, 3));
3435     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3436   } break;
3437   case DM_POLYTOPE_TRI_PRISM_TENSOR: {
3438     PetscInt    numPoints[2]        = {6, 1};
3439     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3440     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3441     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3442     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3443 
3444     PetscCall(DMSetDimension(rdm, 3));
3445     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3446   } break;
3447   case DM_POLYTOPE_QUAD_PRISM_TENSOR: {
3448     PetscInt    numPoints[2]        = {8, 1};
3449     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3450     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3451     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3452     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3453 
3454     PetscCall(DMSetDimension(rdm, 3));
3455     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3456   } break;
3457   case DM_POLYTOPE_PYRAMID: {
3458     PetscInt    numPoints[2]        = {5, 1};
3459     PetscInt    coneSize[6]         = {5, 0, 0, 0, 0, 0};
3460     PetscInt    cones[5]            = {1, 2, 3, 4, 5};
3461     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3462     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, 0.0, 0.0, 1.0};
3463 
3464     PetscCall(DMSetDimension(rdm, 3));
3465     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3466   } break;
3467   default:
3468     SETERRQ(PetscObjectComm((PetscObject)rdm), PETSC_ERR_ARG_WRONG, "Cannot create reference cell for cell type %s", DMPolytopeTypes[ct]);
3469   }
3470   {
3471     PetscInt Nv, v;
3472 
3473     /* Must create the celltype label here so that we do not automatically try to compute the types */
3474     PetscCall(DMCreateLabel(rdm, "celltype"));
3475     PetscCall(DMPlexSetCellType(rdm, 0, ct));
3476     PetscCall(DMPlexGetChart(rdm, NULL, &Nv));
3477     for (v = 1; v < Nv; ++v) PetscCall(DMPlexSetCellType(rdm, v, DM_POLYTOPE_POINT));
3478   }
3479   PetscCall(DMPlexInterpolateInPlace_Internal(rdm));
3480   PetscCall(PetscObjectSetName((PetscObject)rdm, DMPolytopeTypes[ct]));
3481   PetscFunctionReturn(0);
3482 }
3483 
3484 /*@
3485   DMPlexCreateReferenceCell - Create a DMPLEX with the appropriate FEM reference cell
3486 
3487   Collective
3488 
3489   Input Parameters:
3490 + comm - The communicator
3491 - ct   - The cell type of the reference cell
3492 
3493   Output Parameter:
3494 . refdm - The reference cell
3495 
3496   Level: intermediate
3497 
3498 .seealso: `DMPlexCreateReferenceCell()`, `DMPlexCreateBoxMesh()`
3499 @*/
3500 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, DMPolytopeType ct, DM *refdm)
3501 {
3502   PetscFunctionBegin;
3503   PetscCall(DMCreate(comm, refdm));
3504   PetscCall(DMSetType(*refdm, DMPLEX));
3505   PetscCall(DMPlexCreateReferenceCell_Internal(*refdm, ct));
3506   PetscFunctionReturn(0);
3507 }
3508 
3509 static PetscErrorCode DMPlexCreateBoundaryLabel_Private(DM dm, const char name[])
3510 {
3511   DM        plex;
3512   DMLabel   label;
3513   PetscBool hasLabel;
3514 
3515   PetscFunctionBegin;
3516   PetscCall(DMHasLabel(dm, name, &hasLabel));
3517   if (hasLabel) PetscFunctionReturn(0);
3518   PetscCall(DMCreateLabel(dm, name));
3519   PetscCall(DMGetLabel(dm, name, &label));
3520   PetscCall(DMConvert(dm, DMPLEX, &plex));
3521   PetscCall(DMPlexMarkBoundaryFaces(plex, 1, label));
3522   PetscCall(DMPlexLabelComplete(plex, label));
3523   PetscCall(DMDestroy(&plex));
3524   PetscFunctionReturn(0);
3525 }
3526 
3527 /*
3528   We use the last coordinate as the radius, the inner radius is lower[dim-1] and the outer radius is upper[dim-1]. Then we map the first coordinate around the circle.
3529 
3530     (x, y) -> (r, theta) = (x[1], (x[0] - lower[0]) * 2\pi/(upper[0] - lower[0]))
3531 */
3532 static void boxToAnnulus(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
3533 {
3534   const PetscReal low = PetscRealPart(constants[0]);
3535   const PetscReal upp = PetscRealPart(constants[1]);
3536   const PetscReal r   = PetscRealPart(u[1]);
3537   const PetscReal th  = 2. * PETSC_PI * (PetscRealPart(u[0]) - low) / (upp - low);
3538 
3539   f0[0] = r * PetscCosReal(th);
3540   f0[1] = r * PetscSinReal(th);
3541 }
3542 
3543 const char *const DMPlexShapes[] = {"box", "box_surface", "ball", "sphere", "cylinder", "schwarz_p", "gyroid", "doublet", "annulus", "unknown", "DMPlexShape", "DM_SHAPE_", NULL};
3544 
3545 static PetscErrorCode DMPlexCreateFromOptions_Internal(PetscOptionItems *PetscOptionsObject, PetscBool *useCoordSpace, DM dm)
3546 {
3547   DMPlexShape    shape   = DM_SHAPE_BOX;
3548   DMPolytopeType cell    = DM_POLYTOPE_TRIANGLE;
3549   PetscInt       dim     = 2;
3550   PetscBool      simplex = PETSC_TRUE, interpolate = PETSC_TRUE, adjCone = PETSC_FALSE, adjClosure = PETSC_TRUE, refDomain = PETSC_FALSE;
3551   PetscBool      flg, flg2, fflg, bdfflg, nameflg;
3552   MPI_Comm       comm;
3553   char           filename[PETSC_MAX_PATH_LEN]   = "<unspecified>";
3554   char           bdFilename[PETSC_MAX_PATH_LEN] = "<unspecified>";
3555   char           plexname[PETSC_MAX_PATH_LEN]   = "";
3556 
3557   PetscFunctionBegin;
3558   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3559   /* TODO Turn this into a registration interface */
3560   PetscCall(PetscOptionsString("-dm_plex_filename", "File containing a mesh", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &fflg));
3561   PetscCall(PetscOptionsString("-dm_plex_boundary_filename", "File containing a mesh boundary", "DMPlexCreateFromFile", bdFilename, bdFilename, sizeof(bdFilename), &bdfflg));
3562   PetscCall(PetscOptionsString("-dm_plex_name", "Name of the mesh in the file", "DMPlexCreateFromFile", plexname, plexname, sizeof(plexname), &nameflg));
3563   PetscCall(PetscOptionsEnum("-dm_plex_cell", "Cell shape", "", DMPolytopeTypes, (PetscEnum)cell, (PetscEnum *)&cell, NULL));
3564   PetscCall(PetscOptionsBool("-dm_plex_reference_cell_domain", "Use a reference cell domain", "", refDomain, &refDomain, NULL));
3565   PetscCall(PetscOptionsEnum("-dm_plex_shape", "Shape for built-in mesh", "", DMPlexShapes, (PetscEnum)shape, (PetscEnum *)&shape, &flg));
3566   PetscCall(PetscOptionsBoundedInt("-dm_plex_dim", "Topological dimension of the mesh", "DMGetDimension", dim, &dim, &flg, 0));
3567   PetscCheck(!(dim < 0) && !(dim > 3), comm, PETSC_ERR_ARG_OUTOFRANGE, "Dimension %" PetscInt_FMT " should be in [1, 3]", dim);
3568   PetscCall(PetscOptionsBool("-dm_plex_simplex", "Mesh cell shape", "", simplex, &simplex, &flg));
3569   PetscCall(PetscOptionsBool("-dm_plex_interpolate", "Flag to create edges and faces automatically", "", interpolate, &interpolate, &flg));
3570   PetscCall(PetscOptionsBool("-dm_plex_adj_cone", "Set adjacency direction", "DMSetBasicAdjacency", adjCone, &adjCone, &flg));
3571   PetscCall(PetscOptionsBool("-dm_plex_adj_closure", "Set adjacency size", "DMSetBasicAdjacency", adjClosure, &adjClosure, &flg2));
3572   if (flg || flg2) PetscCall(DMSetBasicAdjacency(dm, adjCone, adjClosure));
3573 
3574   switch (cell) {
3575   case DM_POLYTOPE_POINT:
3576   case DM_POLYTOPE_SEGMENT:
3577   case DM_POLYTOPE_POINT_PRISM_TENSOR:
3578   case DM_POLYTOPE_TRIANGLE:
3579   case DM_POLYTOPE_QUADRILATERAL:
3580   case DM_POLYTOPE_TETRAHEDRON:
3581   case DM_POLYTOPE_HEXAHEDRON:
3582     *useCoordSpace = PETSC_TRUE;
3583     break;
3584   default:
3585     *useCoordSpace = PETSC_FALSE;
3586     break;
3587   }
3588 
3589   if (fflg) {
3590     DM dmnew;
3591 
3592     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), filename, plexname, interpolate, &dmnew));
3593     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3594     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3595   } else if (refDomain) {
3596     PetscCall(DMPlexCreateReferenceCell_Internal(dm, cell));
3597   } else if (bdfflg) {
3598     DM bdm, dmnew;
3599 
3600     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), bdFilename, plexname, interpolate, &bdm));
3601     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)bdm, "bd_"));
3602     PetscCall(DMSetFromOptions(bdm));
3603     PetscCall(DMPlexGenerate(bdm, NULL, interpolate, &dmnew));
3604     PetscCall(DMDestroy(&bdm));
3605     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3606     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3607   } else {
3608     PetscCall(PetscObjectSetName((PetscObject)dm, DMPlexShapes[shape]));
3609     switch (shape) {
3610     case DM_SHAPE_BOX:
3611     case DM_SHAPE_ANNULUS: {
3612       PetscInt       faces[3]  = {0, 0, 0};
3613       PetscReal      lower[3]  = {0, 0, 0};
3614       PetscReal      upper[3]  = {1, 1, 1};
3615       DMBoundaryType bdt[3]    = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3616       PetscBool      isAnnular = shape == DM_SHAPE_ANNULUS ? PETSC_TRUE : PETSC_FALSE;
3617       PetscInt       i, n;
3618 
3619       n = dim;
3620       for (i = 0; i < dim; ++i) faces[i] = (dim == 1 ? 1 : 4 - dim);
3621       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3622       n = 3;
3623       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3624       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3625       n = 3;
3626       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3627       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3628       n = 3;
3629       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
3630       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3631 
3632       PetscCheck(!isAnnular || dim == 2, comm, PETSC_ERR_ARG_OUTOFRANGE, "Only two dimensional annuli have been implemented");
3633       if (isAnnular)
3634         for (i = 0; i < dim - 1; ++i) bdt[i] = DM_BOUNDARY_PERIODIC;
3635 
3636       switch (cell) {
3637       case DM_POLYTOPE_TRI_PRISM_TENSOR:
3638         PetscCall(DMPlexCreateWedgeBoxMesh_Internal(dm, faces, lower, upper, bdt));
3639         if (!interpolate) {
3640           DM udm;
3641 
3642           PetscCall(DMPlexUninterpolate(dm, &udm));
3643           PetscCall(DMPlexReplace_Internal(dm, &udm));
3644         }
3645         break;
3646       default:
3647         PetscCall(DMPlexCreateBoxMesh_Internal(dm, dim, simplex, faces, lower, upper, bdt, interpolate));
3648         break;
3649       }
3650       if (isAnnular) {
3651         DM          cdm;
3652         PetscDS     cds;
3653         PetscScalar bounds[2] = {lower[0], upper[0]};
3654 
3655         // Fix coordinates for annular region
3656         PetscCall(DMSetPeriodicity(dm, NULL, NULL, NULL));
3657         PetscCall(DMSetCellCoordinatesLocal(dm, NULL));
3658         PetscCall(DMSetCellCoordinates(dm, NULL));
3659         PetscCall(DMPlexCreateCoordinateSpace(dm, 1, NULL));
3660         PetscCall(DMGetCoordinateDM(dm, &cdm));
3661         PetscCall(DMGetDS(cdm, &cds));
3662         PetscCall(PetscDSSetConstants(cds, 2, bounds));
3663         PetscCall(DMPlexRemapGeometry(dm, 0.0, boxToAnnulus));
3664       }
3665     } break;
3666     case DM_SHAPE_BOX_SURFACE: {
3667       PetscInt  faces[3] = {0, 0, 0};
3668       PetscReal lower[3] = {0, 0, 0};
3669       PetscReal upper[3] = {1, 1, 1};
3670       PetscInt  i, n;
3671 
3672       n = dim + 1;
3673       for (i = 0; i < dim + 1; ++i) faces[i] = (dim + 1 == 1 ? 1 : 4 - (dim + 1));
3674       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3675       n = 3;
3676       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3677       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3678       n = 3;
3679       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3680       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3681       PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(dm, dim + 1, faces, lower, upper, interpolate));
3682     } break;
3683     case DM_SHAPE_SPHERE: {
3684       PetscReal R = 1.0;
3685 
3686       PetscCall(PetscOptionsReal("-dm_plex_sphere_radius", "Radius of the sphere", "", R, &R, &flg));
3687       PetscCall(DMPlexCreateSphereMesh_Internal(dm, dim, simplex, R));
3688     } break;
3689     case DM_SHAPE_BALL: {
3690       PetscReal R = 1.0;
3691 
3692       PetscCall(PetscOptionsReal("-dm_plex_ball_radius", "Radius of the ball", "", R, &R, &flg));
3693       PetscCall(DMPlexCreateBallMesh_Internal(dm, dim, R));
3694     } break;
3695     case DM_SHAPE_CYLINDER: {
3696       DMBoundaryType bdt = DM_BOUNDARY_NONE;
3697       PetscInt       Nw  = 6;
3698 
3699       PetscCall(PetscOptionsEnum("-dm_plex_cylinder_bd", "Boundary type in the z direction", "", DMBoundaryTypes, (PetscEnum)bdt, (PetscEnum *)&bdt, NULL));
3700       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_wedges", "Number of wedges around the cylinder", "", Nw, &Nw, NULL));
3701       switch (cell) {
3702       case DM_POLYTOPE_TRI_PRISM_TENSOR:
3703         PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(dm, Nw, interpolate));
3704         break;
3705       default:
3706         PetscCall(DMPlexCreateHexCylinderMesh_Internal(dm, bdt));
3707         break;
3708       }
3709     } break;
3710     case DM_SHAPE_SCHWARZ_P: // fallthrough
3711     case DM_SHAPE_GYROID: {
3712       PetscInt       extent[3] = {1, 1, 1}, refine = 0, layers = 0, three;
3713       PetscReal      thickness   = 0.;
3714       DMBoundaryType periodic[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3715       DMPlexTPSType  tps_type    = shape == DM_SHAPE_SCHWARZ_P ? DMPLEX_TPS_SCHWARZ_P : DMPLEX_TPS_GYROID;
3716       PetscBool      tps_distribute;
3717       PetscCall(PetscOptionsIntArray("-dm_plex_tps_extent", "Number of replicas for each of three dimensions", NULL, extent, (three = 3, &three), NULL));
3718       PetscCall(PetscOptionsInt("-dm_plex_tps_refine", "Number of refinements", NULL, refine, &refine, NULL));
3719       PetscCall(PetscOptionsEnumArray("-dm_plex_tps_periodic", "Periodicity in each of three dimensions", NULL, DMBoundaryTypes, (PetscEnum *)periodic, (three = 3, &three), NULL));
3720       PetscCall(PetscOptionsInt("-dm_plex_tps_layers", "Number of layers in volumetric extrusion (or zero to not extrude)", NULL, layers, &layers, NULL));
3721       PetscCall(PetscOptionsReal("-dm_plex_tps_thickness", "Thickness of volumetric extrusion", NULL, thickness, &thickness, NULL));
3722       PetscCall(DMPlexDistributeGetDefault(dm, &tps_distribute));
3723       PetscCall(PetscOptionsBool("-dm_plex_tps_distribute", "Distribute the 2D mesh prior to refinement and extrusion", NULL, tps_distribute, &tps_distribute, NULL));
3724       PetscCall(DMPlexCreateTPSMesh_Internal(dm, tps_type, extent, periodic, tps_distribute, refine, layers, thickness));
3725     } break;
3726     case DM_SHAPE_DOUBLET: {
3727       DM        dmnew;
3728       PetscReal rl = 0.0;
3729 
3730       PetscCall(PetscOptionsReal("-dm_plex_doublet_refinementlimit", "Refinement limit", NULL, rl, &rl, NULL));
3731       PetscCall(DMPlexCreateDoublet(PetscObjectComm((PetscObject)dm), dim, simplex, interpolate, rl, &dmnew));
3732       PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3733       PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3734     } break;
3735     default:
3736       SETERRQ(comm, PETSC_ERR_SUP, "Domain shape %s is unsupported", DMPlexShapes[shape]);
3737     }
3738   }
3739   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3740   if (!((PetscObject)dm)->name && nameflg) PetscCall(PetscObjectSetName((PetscObject)dm, plexname));
3741   PetscFunctionReturn(0);
3742 }
3743 
3744 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
3745 {
3746   DM_Plex  *mesh = (DM_Plex *)dm->data;
3747   PetscBool flg, flg2;
3748   char      bdLabel[PETSC_MAX_PATH_LEN];
3749 
3750   PetscFunctionBegin;
3751   /* Handle viewing */
3752   PetscCall(PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL));
3753   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fem", "Debug output level all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL, 0));
3754   PetscCall(PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL));
3755   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL, 0));
3756   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_locate", "Debug output level all point location computations", "DMLocatePoints", 0, &mesh->printLocate, NULL, 0));
3757   PetscCall(DMMonitorSetFromOptions(dm, "-dm_plex_monitor_throughput", "Monitor the simulation throughput", "DMPlexMonitorThroughput", DMPlexMonitorThroughput, NULL, &flg));
3758   if (flg) PetscCall(PetscLogDefaultBegin());
3759   /* Labeling */
3760   PetscCall(PetscOptionsString("-dm_plex_boundary_label", "Label to mark the mesh boundary", "", bdLabel, bdLabel, sizeof(bdLabel), &flg));
3761   if (flg) PetscCall(DMPlexCreateBoundaryLabel_Private(dm, bdLabel));
3762   /* Point Location */
3763   PetscCall(PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL));
3764   /* Partitioning and distribution */
3765   PetscCall(PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL));
3766   /* Generation and remeshing */
3767   PetscCall(PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL));
3768   /* Projection behavior */
3769   PetscCall(PetscOptionsBoundedInt("-dm_plex_max_projection_height", "Maximum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL, 0));
3770   PetscCall(PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL));
3771   /* Checking structure */
3772   {
3773     PetscBool all = PETSC_FALSE;
3774 
3775     PetscCall(PetscOptionsBool("-dm_plex_check_all", "Perform all basic checks", "DMPlexCheck", PETSC_FALSE, &all, NULL));
3776     if (all) {
3777       PetscCall(DMPlexCheck(dm));
3778     } else {
3779       PetscCall(PetscOptionsBool("-dm_plex_check_symmetry", "Check that the adjacency information in the mesh is symmetric", "DMPlexCheckSymmetry", PETSC_FALSE, &flg, &flg2));
3780       if (flg && flg2) PetscCall(DMPlexCheckSymmetry(dm));
3781       PetscCall(PetscOptionsBool("-dm_plex_check_skeleton", "Check that each cell has the correct number of vertices (only for homogeneous simplex or tensor meshes)", "DMPlexCheckSkeleton", PETSC_FALSE, &flg, &flg2));
3782       if (flg && flg2) PetscCall(DMPlexCheckSkeleton(dm, 0));
3783       PetscCall(PetscOptionsBool("-dm_plex_check_faces", "Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type", "DMPlexCheckFaces", PETSC_FALSE, &flg, &flg2));
3784       if (flg && flg2) PetscCall(DMPlexCheckFaces(dm, 0));
3785       PetscCall(PetscOptionsBool("-dm_plex_check_geometry", "Check that cells have positive volume", "DMPlexCheckGeometry", PETSC_FALSE, &flg, &flg2));
3786       if (flg && flg2) PetscCall(DMPlexCheckGeometry(dm));
3787       PetscCall(PetscOptionsBool("-dm_plex_check_pointsf", "Check some necessary conditions for PointSF", "DMPlexCheckPointSF", PETSC_FALSE, &flg, &flg2));
3788       if (flg && flg2) PetscCall(DMPlexCheckPointSF(dm, NULL, PETSC_FALSE));
3789       PetscCall(PetscOptionsBool("-dm_plex_check_interface_cones", "Check points on inter-partition interfaces have conforming order of cone points", "DMPlexCheckInterfaceCones", PETSC_FALSE, &flg, &flg2));
3790       if (flg && flg2) PetscCall(DMPlexCheckInterfaceCones(dm));
3791     }
3792     PetscCall(PetscOptionsBool("-dm_plex_check_cell_shape", "Check cell shape", "DMPlexCheckCellShape", PETSC_FALSE, &flg, &flg2));
3793     if (flg && flg2) PetscCall(DMPlexCheckCellShape(dm, PETSC_TRUE, PETSC_DETERMINE));
3794   }
3795   {
3796     PetscReal scale = 1.0;
3797 
3798     PetscCall(PetscOptionsReal("-dm_plex_scale", "Scale factor for mesh coordinates", "DMPlexScale", scale, &scale, &flg));
3799     if (flg) {
3800       Vec coordinates, coordinatesLocal;
3801 
3802       PetscCall(DMGetCoordinates(dm, &coordinates));
3803       PetscCall(DMGetCoordinatesLocal(dm, &coordinatesLocal));
3804       PetscCall(VecScale(coordinates, scale));
3805       PetscCall(VecScale(coordinatesLocal, scale));
3806     }
3807   }
3808   PetscCall(PetscPartitionerSetFromOptions(mesh->partitioner));
3809   PetscFunctionReturn(0);
3810 }
3811 
3812 PetscErrorCode DMSetFromOptions_Overlap_Plex(DM dm, PetscOptionItems *PetscOptionsObject, PetscInt *overlap)
3813 {
3814   PetscInt  numOvLabels = 16, numOvExLabels = 16;
3815   char     *ovLabelNames[16], *ovExLabelNames[16];
3816   PetscInt  numOvValues = 16, numOvExValues = 16, l;
3817   PetscBool flg;
3818 
3819   PetscFunctionBegin;
3820   PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", *overlap, overlap, NULL, 0));
3821   PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_labels", "List of overlap label names", "DMPlexDistribute", ovLabelNames, &numOvLabels, &flg));
3822   if (!flg) numOvLabels = 0;
3823   if (numOvLabels) {
3824     ((DM_Plex *)dm->data)->numOvLabels = numOvLabels;
3825     for (l = 0; l < numOvLabels; ++l) {
3826       PetscCall(DMGetLabel(dm, ovLabelNames[l], &((DM_Plex *)dm->data)->ovLabels[l]));
3827       PetscCheck(((DM_Plex *)dm->data)->ovLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovLabelNames[l]);
3828       PetscCall(PetscFree(ovLabelNames[l]));
3829     }
3830     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_values", "List of overlap label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovValues, &numOvValues, &flg));
3831     if (!flg) numOvValues = 0;
3832     PetscCheck(numOvLabels == numOvValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvLabels, numOvValues);
3833 
3834     PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_exclude_labels", "List of overlap exclude label names", "DMPlexDistribute", ovExLabelNames, &numOvExLabels, &flg));
3835     if (!flg) numOvExLabels = 0;
3836     ((DM_Plex *)dm->data)->numOvExLabels = numOvExLabels;
3837     for (l = 0; l < numOvExLabels; ++l) {
3838       PetscCall(DMGetLabel(dm, ovExLabelNames[l], &((DM_Plex *)dm->data)->ovExLabels[l]));
3839       PetscCheck(((DM_Plex *)dm->data)->ovExLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovExLabelNames[l]);
3840       PetscCall(PetscFree(ovExLabelNames[l]));
3841     }
3842     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_exclude_values", "List of overlap exclude label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovExValues, &numOvExValues, &flg));
3843     if (!flg) numOvExValues = 0;
3844     PetscCheck(numOvExLabels == numOvExValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of exclude labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvExLabels, numOvExValues);
3845   }
3846   PetscFunctionReturn(0);
3847 }
3848 
3849 static PetscErrorCode DMSetFromOptions_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
3850 {
3851   PetscFunctionList        ordlist;
3852   char                     oname[256];
3853   PetscReal                volume    = -1.0;
3854   PetscInt                 prerefine = 0, refine = 0, r, coarsen = 0, overlap = 0, extLayers = 0, dim;
3855   PetscBool                uniformOrig, created = PETSC_FALSE, uniform = PETSC_TRUE, distribute, interpolate = PETSC_TRUE, coordSpace = PETSC_TRUE, remap = PETSC_TRUE, ghostCells = PETSC_FALSE, isHierarchy, ignoreModel = PETSC_FALSE, flg;
3856   DMPlexReorderDefaultFlag reorder;
3857 
3858   PetscFunctionBegin;
3859   PetscOptionsHeadBegin(PetscOptionsObject, "DMPlex Options");
3860   /* Handle automatic creation */
3861   PetscCall(DMGetDimension(dm, &dim));
3862   if (dim < 0) {
3863     PetscCall(DMPlexCreateFromOptions_Internal(PetscOptionsObject, &coordSpace, dm));
3864     created = PETSC_TRUE;
3865   }
3866   PetscCall(DMGetDimension(dm, &dim));
3867   /* Handle interpolation before distribution */
3868   PetscCall(PetscOptionsBool("-dm_plex_interpolate_pre", "Flag to interpolate mesh before distribution", "", interpolate, &interpolate, &flg));
3869   if (flg) {
3870     DMPlexInterpolatedFlag interpolated;
3871 
3872     PetscCall(DMPlexIsInterpolated(dm, &interpolated));
3873     if (interpolated == DMPLEX_INTERPOLATED_FULL && !interpolate) {
3874       DM udm;
3875 
3876       PetscCall(DMPlexUninterpolate(dm, &udm));
3877       PetscCall(DMPlexReplace_Internal(dm, &udm));
3878     } else if (interpolated != DMPLEX_INTERPOLATED_FULL && interpolate) {
3879       DM idm;
3880 
3881       PetscCall(DMPlexInterpolate(dm, &idm));
3882       PetscCall(DMPlexReplace_Internal(dm, &idm));
3883     }
3884   }
3885   /* Handle DMPlex refinement before distribution */
3886   PetscCall(PetscOptionsBool("-dm_refine_ignore_model", "Flag to ignore the geometry model when refining", "DMCreate", ignoreModel, &ignoreModel, &flg));
3887   if (flg) ((DM_Plex *)dm->data)->ignoreModel = ignoreModel;
3888   PetscCall(DMPlexGetRefinementUniform(dm, &uniformOrig));
3889   PetscCall(PetscOptionsBoundedInt("-dm_refine_pre", "The number of refinements before distribution", "DMCreate", prerefine, &prerefine, NULL, 0));
3890   PetscCall(PetscOptionsBool("-dm_refine_remap_pre", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
3891   PetscCall(PetscOptionsBool("-dm_refine_uniform_pre", "Flag for uniform refinement before distribution", "DMCreate", uniform, &uniform, &flg));
3892   if (flg) PetscCall(DMPlexSetRefinementUniform(dm, uniform));
3893   PetscCall(PetscOptionsReal("-dm_refine_volume_limit_pre", "The maximum cell volume after refinement before distribution", "DMCreate", volume, &volume, &flg));
3894   if (flg) {
3895     PetscCall(DMPlexSetRefinementUniform(dm, PETSC_FALSE));
3896     PetscCall(DMPlexSetRefinementLimit(dm, volume));
3897     prerefine = PetscMax(prerefine, 1);
3898   }
3899   for (r = 0; r < prerefine; ++r) {
3900     DM             rdm;
3901     PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
3902 
3903     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3904     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
3905     PetscCall(DMPlexReplace_Internal(dm, &rdm));
3906     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3907     if (coordFunc && remap) {
3908       PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
3909       ((DM_Plex *)dm->data)->coordFunc = coordFunc;
3910     }
3911   }
3912   PetscCall(DMPlexSetRefinementUniform(dm, uniformOrig));
3913   /* Handle DMPlex extrusion before distribution */
3914   PetscCall(PetscOptionsBoundedInt("-dm_extrude", "The number of layers to extrude", "", extLayers, &extLayers, NULL, 0));
3915   if (extLayers) {
3916     DM edm;
3917 
3918     PetscCall(DMExtrude(dm, extLayers, &edm));
3919     PetscCall(DMPlexReplace_Internal(dm, &edm));
3920     ((DM_Plex *)dm->data)->coordFunc = NULL;
3921     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3922     extLayers = 0;
3923   }
3924   /* Handle DMPlex reordering before distribution */
3925   PetscCall(DMPlexReorderGetDefault(dm, &reorder));
3926   PetscCall(MatGetOrderingList(&ordlist));
3927   PetscCall(PetscStrncpy(oname, MATORDERINGNATURAL, sizeof(oname)));
3928   PetscCall(PetscOptionsFList("-dm_plex_reorder", "Set mesh reordering type", "DMPlexGetOrdering", ordlist, MATORDERINGNATURAL, oname, sizeof(oname), &flg));
3929   if (reorder == DMPLEX_REORDER_DEFAULT_TRUE || flg) {
3930     DM pdm;
3931     IS perm;
3932 
3933     PetscCall(DMPlexGetOrdering(dm, oname, NULL, &perm));
3934     PetscCall(DMPlexPermute(dm, perm, &pdm));
3935     PetscCall(ISDestroy(&perm));
3936     PetscCall(DMPlexReplace_Internal(dm, &pdm));
3937     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3938   }
3939   /* Handle DMPlex distribution */
3940   PetscCall(DMPlexDistributeGetDefault(dm, &distribute));
3941   PetscCall(PetscOptionsBool("-dm_distribute", "Flag to redistribute a mesh among processes", "DMPlexDistribute", distribute, &distribute, NULL));
3942   PetscCall(DMSetFromOptions_Overlap_Plex(dm, PetscOptionsObject, &overlap));
3943   if (distribute) {
3944     DM               pdm = NULL;
3945     PetscPartitioner part;
3946 
3947     PetscCall(DMPlexGetPartitioner(dm, &part));
3948     PetscCall(PetscPartitionerSetFromOptions(part));
3949     PetscCall(DMPlexDistribute(dm, overlap, NULL, &pdm));
3950     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
3951   }
3952   /* Create coordinate space */
3953   if (created) {
3954     DM_Plex  *mesh   = (DM_Plex *)dm->data;
3955     PetscInt  degree = 1;
3956     PetscBool flg;
3957 
3958     PetscCall(PetscOptionsBool("-dm_coord_space", "Use an FEM space for coordinates", "", coordSpace, &coordSpace, &flg));
3959     PetscCall(PetscOptionsInt("-dm_coord_petscspace_degree", "FEM degree for coordinate space", "", degree, &degree, NULL));
3960     if (coordSpace) PetscCall(DMPlexCreateCoordinateSpace(dm, degree, mesh->coordFunc));
3961     if (flg && !coordSpace) {
3962       DM           cdm;
3963       PetscDS      cds;
3964       PetscObject  obj;
3965       PetscClassId id;
3966 
3967       PetscCall(DMGetCoordinateDM(dm, &cdm));
3968       PetscCall(DMGetDS(cdm, &cds));
3969       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
3970       PetscCall(PetscObjectGetClassId(obj, &id));
3971       if (id == PETSCFE_CLASSID) {
3972         PetscContainer dummy;
3973 
3974         PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &dummy));
3975         PetscCall(PetscObjectSetName((PetscObject)dummy, "coordinates"));
3976         PetscCall(DMSetField(cdm, 0, NULL, (PetscObject)dummy));
3977         PetscCall(PetscContainerDestroy(&dummy));
3978         PetscCall(DMClearDS(cdm));
3979       }
3980       mesh->coordFunc = NULL;
3981     }
3982     PetscCall(PetscOptionsBool("-dm_sparse_localize", "Localize only necessary cells", "", dm->sparseLocalize, &dm->sparseLocalize, &flg));
3983     PetscCall(DMLocalizeCoordinates(dm));
3984   }
3985   /* Handle DMPlex refinement */
3986   remap = PETSC_TRUE;
3987   PetscCall(PetscOptionsBoundedInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL, 0));
3988   PetscCall(PetscOptionsBool("-dm_refine_remap", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
3989   PetscCall(PetscOptionsBoundedInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy, 0));
3990   if (refine) PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3991   if (refine && isHierarchy) {
3992     DM *dms, coarseDM;
3993 
3994     PetscCall(DMGetCoarseDM(dm, &coarseDM));
3995     PetscCall(PetscObjectReference((PetscObject)coarseDM));
3996     PetscCall(PetscMalloc1(refine, &dms));
3997     PetscCall(DMRefineHierarchy(dm, refine, dms));
3998     /* Total hack since we do not pass in a pointer */
3999     PetscCall(DMPlexSwap_Static(dm, dms[refine - 1]));
4000     if (refine == 1) {
4001       PetscCall(DMSetCoarseDM(dm, dms[0]));
4002       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4003     } else {
4004       PetscCall(DMSetCoarseDM(dm, dms[refine - 2]));
4005       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4006       PetscCall(DMSetCoarseDM(dms[0], dms[refine - 1]));
4007       PetscCall(DMPlexSetRegularRefinement(dms[0], PETSC_TRUE));
4008     }
4009     PetscCall(DMSetCoarseDM(dms[refine - 1], coarseDM));
4010     PetscCall(PetscObjectDereference((PetscObject)coarseDM));
4011     /* Free DMs */
4012     for (r = 0; r < refine; ++r) {
4013       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4014       PetscCall(DMDestroy(&dms[r]));
4015     }
4016     PetscCall(PetscFree(dms));
4017   } else {
4018     for (r = 0; r < refine; ++r) {
4019       DM             rdm;
4020       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4021 
4022       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4023       PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
4024       /* Total hack since we do not pass in a pointer */
4025       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4026       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4027       if (coordFunc && remap) {
4028         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4029         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4030       }
4031     }
4032   }
4033   /* Handle DMPlex coarsening */
4034   PetscCall(PetscOptionsBoundedInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL, 0));
4035   PetscCall(PetscOptionsBoundedInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy, 0));
4036   if (coarsen && isHierarchy) {
4037     DM *dms;
4038 
4039     PetscCall(PetscMalloc1(coarsen, &dms));
4040     PetscCall(DMCoarsenHierarchy(dm, coarsen, dms));
4041     /* Free DMs */
4042     for (r = 0; r < coarsen; ++r) {
4043       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4044       PetscCall(DMDestroy(&dms[r]));
4045     }
4046     PetscCall(PetscFree(dms));
4047   } else {
4048     for (r = 0; r < coarsen; ++r) {
4049       DM             cdm;
4050       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4051 
4052       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4053       PetscCall(DMCoarsen(dm, PetscObjectComm((PetscObject)dm), &cdm));
4054       /* Total hack since we do not pass in a pointer */
4055       PetscCall(DMPlexReplace_Internal(dm, &cdm));
4056       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4057       if (coordFunc) {
4058         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4059         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4060       }
4061     }
4062   }
4063   /* Handle ghost cells */
4064   PetscCall(PetscOptionsBool("-dm_plex_create_fv_ghost_cells", "Flag to create finite volume ghost cells on the boundary", "DMCreate", ghostCells, &ghostCells, NULL));
4065   if (ghostCells) {
4066     DM   gdm;
4067     char lname[PETSC_MAX_PATH_LEN];
4068 
4069     lname[0] = '\0';
4070     PetscCall(PetscOptionsString("-dm_plex_fv_ghost_cells_label", "Label name for ghost cells boundary", "DMCreate", lname, lname, sizeof(lname), &flg));
4071     PetscCall(DMPlexConstructGhostCells(dm, flg ? lname : NULL, NULL, &gdm));
4072     PetscCall(DMPlexReplace_Internal(dm, &gdm));
4073   }
4074   /* Handle 1D order */
4075   if (reorder != DMPLEX_REORDER_DEFAULT_FALSE && dim == 1) {
4076     DM           cdm, rdm;
4077     PetscDS      cds;
4078     PetscObject  obj;
4079     PetscClassId id = PETSC_OBJECT_CLASSID;
4080     IS           perm;
4081     PetscInt     Nf;
4082     PetscBool    distributed;
4083 
4084     PetscCall(DMPlexIsDistributed(dm, &distributed));
4085     PetscCall(DMGetCoordinateDM(dm, &cdm));
4086     PetscCall(DMGetDS(cdm, &cds));
4087     PetscCall(PetscDSGetNumFields(cds, &Nf));
4088     if (Nf) {
4089       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
4090       PetscCall(PetscObjectGetClassId(obj, &id));
4091     }
4092     if (!distributed && id != PETSCFE_CLASSID) {
4093       PetscCall(DMPlexGetOrdering1D(dm, &perm));
4094       PetscCall(DMPlexPermute(dm, perm, &rdm));
4095       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4096       PetscCall(ISDestroy(&perm));
4097     }
4098   }
4099   /* Handle */
4100   PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4101   PetscOptionsHeadEnd();
4102   PetscFunctionReturn(0);
4103 }
4104 
4105 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm, Vec *vec)
4106 {
4107   PetscFunctionBegin;
4108   PetscCall(DMCreateGlobalVector_Section_Private(dm, vec));
4109   /* PetscCall(VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM)); */
4110   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex));
4111   PetscCall(VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void))VecView_Plex_Native));
4112   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex));
4113   PetscCall(VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void))VecLoad_Plex_Native));
4114   PetscFunctionReturn(0);
4115 }
4116 
4117 static PetscErrorCode DMCreateLocalVector_Plex(DM dm, Vec *vec)
4118 {
4119   PetscFunctionBegin;
4120   PetscCall(DMCreateLocalVector_Section_Private(dm, vec));
4121   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex_Local));
4122   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex_Local));
4123   PetscFunctionReturn(0);
4124 }
4125 
4126 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
4127 {
4128   PetscInt depth, d;
4129 
4130   PetscFunctionBegin;
4131   PetscCall(DMPlexGetDepth(dm, &depth));
4132   if (depth == 1) {
4133     PetscCall(DMGetDimension(dm, &d));
4134     if (dim == 0) PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4135     else if (dim == d) PetscCall(DMPlexGetDepthStratum(dm, 1, pStart, pEnd));
4136     else {
4137       *pStart = 0;
4138       *pEnd   = 0;
4139     }
4140   } else {
4141     PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4142   }
4143   PetscFunctionReturn(0);
4144 }
4145 
4146 static PetscErrorCode DMGetNeighbors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
4147 {
4148   PetscSF            sf;
4149   PetscInt           niranks, njranks, n;
4150   const PetscMPIInt *iranks, *jranks;
4151   DM_Plex           *data = (DM_Plex *)dm->data;
4152 
4153   PetscFunctionBegin;
4154   PetscCall(DMGetPointSF(dm, &sf));
4155   if (!data->neighbors) {
4156     PetscCall(PetscSFSetUp(sf));
4157     PetscCall(PetscSFGetRootRanks(sf, &njranks, &jranks, NULL, NULL, NULL));
4158     PetscCall(PetscSFGetLeafRanks(sf, &niranks, &iranks, NULL, NULL));
4159     PetscCall(PetscMalloc1(njranks + niranks + 1, &data->neighbors));
4160     PetscCall(PetscArraycpy(data->neighbors + 1, jranks, njranks));
4161     PetscCall(PetscArraycpy(data->neighbors + njranks + 1, iranks, niranks));
4162     n = njranks + niranks;
4163     PetscCall(PetscSortRemoveDupsMPIInt(&n, data->neighbors + 1));
4164     /* The following cast should never fail: can't have more neighbors than PETSC_MPI_INT_MAX */
4165     PetscCall(PetscMPIIntCast(n, data->neighbors));
4166   }
4167   if (nranks) *nranks = data->neighbors[0];
4168   if (ranks) {
4169     if (data->neighbors[0]) *ranks = data->neighbors + 1;
4170     else *ranks = NULL;
4171   }
4172   PetscFunctionReturn(0);
4173 }
4174 
4175 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM, DM, Mat, Vec, Vec);
4176 
4177 static PetscErrorCode DMInitialize_Plex(DM dm)
4178 {
4179   PetscFunctionBegin;
4180   dm->ops->view                      = DMView_Plex;
4181   dm->ops->load                      = DMLoad_Plex;
4182   dm->ops->setfromoptions            = DMSetFromOptions_Plex;
4183   dm->ops->clone                     = DMClone_Plex;
4184   dm->ops->setup                     = DMSetUp_Plex;
4185   dm->ops->createlocalsection        = DMCreateLocalSection_Plex;
4186   dm->ops->createdefaultconstraints  = DMCreateDefaultConstraints_Plex;
4187   dm->ops->createglobalvector        = DMCreateGlobalVector_Plex;
4188   dm->ops->createlocalvector         = DMCreateLocalVector_Plex;
4189   dm->ops->getlocaltoglobalmapping   = NULL;
4190   dm->ops->createfieldis             = NULL;
4191   dm->ops->createcoordinatedm        = DMCreateCoordinateDM_Plex;
4192   dm->ops->createcoordinatefield     = DMCreateCoordinateField_Plex;
4193   dm->ops->getcoloring               = NULL;
4194   dm->ops->creatematrix              = DMCreateMatrix_Plex;
4195   dm->ops->createinterpolation       = DMCreateInterpolation_Plex;
4196   dm->ops->createmassmatrix          = DMCreateMassMatrix_Plex;
4197   dm->ops->createmassmatrixlumped    = DMCreateMassMatrixLumped_Plex;
4198   dm->ops->createinjection           = DMCreateInjection_Plex;
4199   dm->ops->refine                    = DMRefine_Plex;
4200   dm->ops->coarsen                   = DMCoarsen_Plex;
4201   dm->ops->refinehierarchy           = DMRefineHierarchy_Plex;
4202   dm->ops->coarsenhierarchy          = DMCoarsenHierarchy_Plex;
4203   dm->ops->extrude                   = DMExtrude_Plex;
4204   dm->ops->globaltolocalbegin        = NULL;
4205   dm->ops->globaltolocalend          = NULL;
4206   dm->ops->localtoglobalbegin        = NULL;
4207   dm->ops->localtoglobalend          = NULL;
4208   dm->ops->destroy                   = DMDestroy_Plex;
4209   dm->ops->createsubdm               = DMCreateSubDM_Plex;
4210   dm->ops->createsuperdm             = DMCreateSuperDM_Plex;
4211   dm->ops->getdimpoints              = DMGetDimPoints_Plex;
4212   dm->ops->locatepoints              = DMLocatePoints_Plex;
4213   dm->ops->projectfunctionlocal      = DMProjectFunctionLocal_Plex;
4214   dm->ops->projectfunctionlabellocal = DMProjectFunctionLabelLocal_Plex;
4215   dm->ops->projectfieldlocal         = DMProjectFieldLocal_Plex;
4216   dm->ops->projectfieldlabellocal    = DMProjectFieldLabelLocal_Plex;
4217   dm->ops->projectbdfieldlabellocal  = DMProjectBdFieldLabelLocal_Plex;
4218   dm->ops->computel2diff             = DMComputeL2Diff_Plex;
4219   dm->ops->computel2gradientdiff     = DMComputeL2GradientDiff_Plex;
4220   dm->ops->computel2fielddiff        = DMComputeL2FieldDiff_Plex;
4221   dm->ops->getneighbors              = DMGetNeighbors_Plex;
4222   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertBoundaryValues_C", DMPlexInsertBoundaryValues_Plex));
4223   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertTimeDerviativeBoundaryValues_C", DMPlexInsertTimeDerivativeBoundaryValues_Plex));
4224   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMSetUpGLVisViewer_C", DMSetUpGLVisViewer_Plex));
4225   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMCreateNeumannOverlap_C", DMCreateNeumannOverlap_Plex));
4226   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4227   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeGetDefault_C", DMPlexDistributeGetDefault_Plex));
4228   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeSetDefault_C", DMPlexDistributeSetDefault_Plex));
4229   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderGetDefault_C", DMPlexReorderGetDefault_Plex));
4230   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderSetDefault_C", DMPlexReorderSetDefault_Plex));
4231   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMInterpolateSolution_C", DMInterpolateSolution_Plex));
4232   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4233   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetOverlap_C", DMPlexSetOverlap_Plex));
4234   PetscFunctionReturn(0);
4235 }
4236 
4237 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
4238 {
4239   DM_Plex *mesh = (DM_Plex *)dm->data;
4240 
4241   PetscFunctionBegin;
4242   mesh->refct++;
4243   (*newdm)->data = mesh;
4244   PetscCall(PetscObjectChangeTypeName((PetscObject)*newdm, DMPLEX));
4245   PetscCall(DMInitialize_Plex(*newdm));
4246   PetscFunctionReturn(0);
4247 }
4248 
4249 /*MC
4250   DMPLEX = "plex" - A DM object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
4251                     In the local representation, Vecs contain all unknowns in the interior and shared boundary. This is
4252                     specified by a PetscSection object. Ownership in the global representation is determined by
4253                     ownership of the underlying DMPlex points. This is specified by another PetscSection object.
4254 
4255   Options Database Keys:
4256 + -dm_refine_pre                     - Refine mesh before distribution
4257 + -dm_refine_uniform_pre             - Choose uniform or generator-based refinement
4258 + -dm_refine_volume_limit_pre        - Cell volume limit after pre-refinement using generator
4259 . -dm_distribute                     - Distribute mesh across processes
4260 . -dm_distribute_overlap             - Number of cells to overlap for distribution
4261 . -dm_refine                         - Refine mesh after distribution
4262 . -dm_plex_hash_location             - Use grid hashing for point location
4263 . -dm_plex_hash_box_faces <n,m,p>    - The number of divisions in each direction of the grid hash
4264 . -dm_plex_partition_balance         - Attempt to evenly divide points on partition boundary between processes
4265 . -dm_plex_remesh_bd                 - Allow changes to the boundary on remeshing
4266 . -dm_plex_max_projection_height     - Maximum mesh point height used to project locally
4267 . -dm_plex_regular_refinement        - Use special nested projection algorithm for regular refinement
4268 . -dm_plex_check_all                 - Perform all shecks below
4269 . -dm_plex_check_symmetry            - Check that the adjacency information in the mesh is symmetric
4270 . -dm_plex_check_skeleton <celltype> - Check that each cell has the correct number of vertices
4271 . -dm_plex_check_faces <celltype>    - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
4272 . -dm_plex_check_geometry            - Check that cells have positive volume
4273 . -dm_view :mesh.tex:ascii_latex     - View the mesh in LaTeX/TikZ
4274 . -dm_plex_view_scale <num>          - Scale the TikZ
4275 - -dm_plex_print_fem <num>           - View FEM assembly information, such as element vectors and matrices
4276 
4277   Level: intermediate
4278 
4279 .seealso: `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`
4280 M*/
4281 
4282 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
4283 {
4284   DM_Plex *mesh;
4285   PetscInt unit;
4286 
4287   PetscFunctionBegin;
4288   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4289   PetscCall(PetscNew(&mesh));
4290   dm->data = mesh;
4291 
4292   mesh->refct = 1;
4293   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection));
4294   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection));
4295   mesh->refinementUniform      = PETSC_TRUE;
4296   mesh->refinementLimit        = -1.0;
4297   mesh->distDefault            = PETSC_TRUE;
4298   mesh->reorderDefault         = DMPLEX_REORDER_DEFAULT_NOTSET;
4299   mesh->distributionName       = NULL;
4300   mesh->interpolated           = DMPLEX_INTERPOLATED_INVALID;
4301   mesh->interpolatedCollective = DMPLEX_INTERPOLATED_INVALID;
4302 
4303   PetscCall(PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner));
4304   mesh->remeshBd = PETSC_FALSE;
4305 
4306   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
4307 
4308   mesh->depthState    = -1;
4309   mesh->celltypeState = -1;
4310   mesh->printTol      = 1.0e-10;
4311 
4312   PetscCall(DMInitialize_Plex(dm));
4313   PetscFunctionReturn(0);
4314 }
4315 
4316 /*@
4317   DMPlexCreate - Creates a DMPlex object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
4318 
4319   Collective
4320 
4321   Input Parameter:
4322 . comm - The communicator for the DMPlex object
4323 
4324   Output Parameter:
4325 . mesh  - The DMPlex object
4326 
4327   Level: beginner
4328 
4329 @*/
4330 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
4331 {
4332   PetscFunctionBegin;
4333   PetscValidPointer(mesh, 2);
4334   PetscCall(DMCreate(comm, mesh));
4335   PetscCall(DMSetType(*mesh, DMPLEX));
4336   PetscFunctionReturn(0);
4337 }
4338 
4339 /*@C
4340   DMPlexBuildFromCellListParallel - Build distributed DMPLEX topology from a list of vertices for each cell (common mesh generator output)
4341 
4342   Input Parameters:
4343 + dm - The DM
4344 . numCells - The number of cells owned by this process
4345 . numVertices - The number of vertices to be owned by this process, or PETSC_DECIDE
4346 . NVertices - The global number of vertices, or PETSC_DETERMINE
4347 . numCorners - The number of vertices for each cell
4348 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4349 
4350   Output Parameters:
4351 + vertexSF - (Optional) SF describing complete vertex ownership
4352 - verticesAdjSaved - (Optional) vertex adjacency array
4353 
4354   Notes:
4355   Two triangles sharing a face
4356 $
4357 $        2
4358 $      / | \
4359 $     /  |  \
4360 $    /   |   \
4361 $   0  0 | 1  3
4362 $    \   |   /
4363 $     \  |  /
4364 $      \ | /
4365 $        1
4366 would have input
4367 $  numCells = 2, numVertices = 4
4368 $  cells = [0 1 2  1 3 2]
4369 $
4370 which would result in the DMPlex
4371 $
4372 $        4
4373 $      / | \
4374 $     /  |  \
4375 $    /   |   \
4376 $   2  0 | 1  5
4377 $    \   |   /
4378 $     \  |  /
4379 $      \ | /
4380 $        3
4381 
4382   Vertices are implicitly numbered consecutively 0,...,NVertices.
4383   Each rank owns a chunk of numVertices consecutive vertices.
4384   If numVertices is PETSC_DECIDE, PETSc will distribute them as evenly as possible using PetscLayout.
4385   If NVertices is PETSC_DETERMINE and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
4386   If only NVertices is PETSC_DETERMINE, it is computed as the sum of numVertices over all ranks.
4387 
4388   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
4389 
4390   Not currently supported in Fortran.
4391 
4392   Level: advanced
4393 
4394 .seealso: `DMPlexBuildFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildCoordinatesFromCellListParallel()`
4395 @*/
4396 PetscErrorCode DMPlexBuildFromCellListParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, const PetscInt cells[], PetscSF *vertexSF, PetscInt **verticesAdjSaved)
4397 {
4398   PetscSF     sfPoint;
4399   PetscLayout layout;
4400   PetscInt    numVerticesAdj, *verticesAdj, *cones, c, p;
4401 
4402   PetscFunctionBegin;
4403   PetscValidLogicalCollectiveInt(dm, NVertices, 4);
4404   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4405   /* Get/check global number of vertices */
4406   {
4407     PetscInt       NVerticesInCells, i;
4408     const PetscInt len = numCells * numCorners;
4409 
4410     /* NVerticesInCells = max(cells) + 1 */
4411     NVerticesInCells = PETSC_MIN_INT;
4412     for (i = 0; i < len; i++)
4413       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4414     ++NVerticesInCells;
4415     PetscCallMPI(MPI_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
4416 
4417     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
4418     else
4419       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
4420   }
4421   /* Count locally unique vertices */
4422   {
4423     PetscHSetI vhash;
4424     PetscInt   off = 0;
4425 
4426     PetscCall(PetscHSetICreate(&vhash));
4427     for (c = 0; c < numCells; ++c) {
4428       for (p = 0; p < numCorners; ++p) PetscCall(PetscHSetIAdd(vhash, cells[c * numCorners + p]));
4429     }
4430     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
4431     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
4432     else verticesAdj = *verticesAdjSaved;
4433     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
4434     PetscCall(PetscHSetIDestroy(&vhash));
4435     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
4436   }
4437   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
4438   /* Create cones */
4439   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
4440   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
4441   PetscCall(DMSetUp(dm));
4442   PetscCall(DMPlexGetCones(dm, &cones));
4443   for (c = 0; c < numCells; ++c) {
4444     for (p = 0; p < numCorners; ++p) {
4445       const PetscInt gv = cells[c * numCorners + p];
4446       PetscInt       lv;
4447 
4448       /* Positions within verticesAdj form 0-based local vertex numbering;
4449          we need to shift it by numCells to get correct DAG points (cells go first) */
4450       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
4451       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
4452       cones[c * numCorners + p] = lv + numCells;
4453     }
4454   }
4455   /* Build point sf */
4456   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
4457   PetscCall(PetscLayoutSetSize(layout, NVertices));
4458   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
4459   PetscCall(PetscLayoutSetBlockSize(layout, 1));
4460   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
4461   PetscCall(PetscLayoutDestroy(&layout));
4462   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
4463   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
4464   if (dm->sf) {
4465     const char *prefix;
4466 
4467     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
4468     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
4469   }
4470   PetscCall(DMSetPointSF(dm, sfPoint));
4471   PetscCall(PetscSFDestroy(&sfPoint));
4472   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)(*vertexSF), "Vertex Ownership SF"));
4473   /* Fill in the rest of the topology structure */
4474   PetscCall(DMPlexSymmetrize(dm));
4475   PetscCall(DMPlexStratify(dm));
4476   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4477   PetscFunctionReturn(0);
4478 }
4479 
4480 /*@C
4481   DMPlexBuildCoordinatesFromCellListParallel - Build DM coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4482 
4483   Input Parameters:
4484 + dm - The DM
4485 . spaceDim - The spatial dimension used for coordinates
4486 . sfVert - SF describing complete vertex ownership
4487 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4488 
4489   Level: advanced
4490 
4491   Notes:
4492   Not currently supported in Fortran.
4493 
4494 .seealso: `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellListParallel()`
4495 @*/
4496 PetscErrorCode DMPlexBuildCoordinatesFromCellListParallel(DM dm, PetscInt spaceDim, PetscSF sfVert, const PetscReal vertexCoords[])
4497 {
4498   PetscSection coordSection;
4499   Vec          coordinates;
4500   PetscScalar *coords;
4501   PetscInt     numVertices, numVerticesAdj, coordSize, v, vStart, vEnd;
4502 
4503   PetscFunctionBegin;
4504   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4505   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
4506   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4507   PetscCall(DMSetCoordinateDim(dm, spaceDim));
4508   PetscCall(PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL));
4509   PetscCheck(vEnd - vStart == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Supplied sfVert has wrong number of leaves = %" PetscInt_FMT " != %" PetscInt_FMT " = vEnd - vStart", numVerticesAdj, vEnd - vStart);
4510   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4511   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4512   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
4513   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
4514   for (v = vStart; v < vEnd; ++v) {
4515     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
4516     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
4517   }
4518   PetscCall(PetscSectionSetUp(coordSection));
4519   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4520   PetscCall(VecCreate(PetscObjectComm((PetscObject)dm), &coordinates));
4521   PetscCall(VecSetBlockSize(coordinates, spaceDim));
4522   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4523   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4524   PetscCall(VecSetType(coordinates, VECSTANDARD));
4525   PetscCall(VecGetArray(coordinates, &coords));
4526   {
4527     MPI_Datatype coordtype;
4528 
4529     /* Need a temp buffer for coords if we have complex/single */
4530     PetscCallMPI(MPI_Type_contiguous(spaceDim, MPIU_SCALAR, &coordtype));
4531     PetscCallMPI(MPI_Type_commit(&coordtype));
4532 #if defined(PETSC_USE_COMPLEX)
4533     {
4534       PetscScalar *svertexCoords;
4535       PetscInt     i;
4536       PetscCall(PetscMalloc1(numVertices * spaceDim, &svertexCoords));
4537       for (i = 0; i < numVertices * spaceDim; i++) svertexCoords[i] = vertexCoords[i];
4538       PetscCall(PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4539       PetscCall(PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4540       PetscCall(PetscFree(svertexCoords));
4541     }
4542 #else
4543     PetscCall(PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4544     PetscCall(PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4545 #endif
4546     PetscCallMPI(MPI_Type_free(&coordtype));
4547   }
4548   PetscCall(VecRestoreArray(coordinates, &coords));
4549   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4550   PetscCall(VecDestroy(&coordinates));
4551   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4552   PetscFunctionReturn(0);
4553 }
4554 
4555 /*@
4556   DMPlexCreateFromCellListParallelPetsc - Create distributed DMPLEX from a list of vertices for each cell (common mesh generator output)
4557 
4558   Input Parameters:
4559 + comm - The communicator
4560 . dim - The topological dimension of the mesh
4561 . numCells - The number of cells owned by this process
4562 . numVertices - The number of vertices owned by this process, or PETSC_DECIDE
4563 . NVertices - The global number of vertices, or PETSC_DECIDE
4564 . numCorners - The number of vertices for each cell
4565 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4566 . cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4567 . spaceDim - The spatial dimension used for coordinates
4568 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4569 
4570   Output Parameters:
4571 + dm - The DM
4572 . vertexSF - (Optional) SF describing complete vertex ownership
4573 - verticesAdjSaved - (Optional) vertex adjacency array
4574 
4575   Notes:
4576   This function is just a convenient sequence of DMCreate(), DMSetType(), DMSetDimension(),
4577   DMPlexBuildFromCellListParallel(), DMPlexInterpolate(), DMPlexBuildCoordinatesFromCellListParallel()
4578 
4579   See DMPlexBuildFromCellListParallel() for an example and details about the topology-related parameters.
4580   See DMPlexBuildCoordinatesFromCellListParallel() for details about the geometry-related parameters.
4581 
4582   Level: intermediate
4583 
4584 .seealso: `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
4585 @*/
4586 PetscErrorCode DMPlexCreateFromCellListParallelPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, PetscInt **verticesAdj, DM *dm)
4587 {
4588   PetscSF sfVert;
4589 
4590   PetscFunctionBegin;
4591   PetscCall(DMCreate(comm, dm));
4592   PetscCall(DMSetType(*dm, DMPLEX));
4593   PetscValidLogicalCollectiveInt(*dm, dim, 2);
4594   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
4595   PetscCall(DMSetDimension(*dm, dim));
4596   PetscCall(DMPlexBuildFromCellListParallel(*dm, numCells, numVertices, NVertices, numCorners, cells, &sfVert, verticesAdj));
4597   if (interpolate) {
4598     DM idm;
4599 
4600     PetscCall(DMPlexInterpolate(*dm, &idm));
4601     PetscCall(DMDestroy(dm));
4602     *dm = idm;
4603   }
4604   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
4605   if (vertexSF) *vertexSF = sfVert;
4606   else PetscCall(PetscSFDestroy(&sfVert));
4607   PetscFunctionReturn(0);
4608 }
4609 
4610 /*@C
4611   DMPlexBuildFromCellList - Build DMPLEX topology from a list of vertices for each cell (common mesh generator output)
4612 
4613   Input Parameters:
4614 + dm - The DM
4615 . numCells - The number of cells owned by this process
4616 . numVertices - The number of vertices owned by this process, or PETSC_DETERMINE
4617 . numCorners - The number of vertices for each cell
4618 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4619 
4620   Level: advanced
4621 
4622   Notes:
4623   Two triangles sharing a face
4624 $
4625 $        2
4626 $      / | \
4627 $     /  |  \
4628 $    /   |   \
4629 $   0  0 | 1  3
4630 $    \   |   /
4631 $     \  |  /
4632 $      \ | /
4633 $        1
4634 would have input
4635 $  numCells = 2, numVertices = 4
4636 $  cells = [0 1 2  1 3 2]
4637 $
4638 which would result in the DMPlex
4639 $
4640 $        4
4641 $      / | \
4642 $     /  |  \
4643 $    /   |   \
4644 $   2  0 | 1  5
4645 $    \   |   /
4646 $     \  |  /
4647 $      \ | /
4648 $        3
4649 
4650   If numVertices is PETSC_DETERMINE, it is computed by PETSc as the maximum vertex index in cells + 1.
4651 
4652   Not currently supported in Fortran.
4653 
4654 .seealso: `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListPetsc()`
4655 @*/
4656 PetscErrorCode DMPlexBuildFromCellList(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const PetscInt cells[])
4657 {
4658   PetscInt *cones, c, p, dim;
4659 
4660   PetscFunctionBegin;
4661   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4662   PetscCall(DMGetDimension(dm, &dim));
4663   /* Get/check global number of vertices */
4664   {
4665     PetscInt       NVerticesInCells, i;
4666     const PetscInt len = numCells * numCorners;
4667 
4668     /* NVerticesInCells = max(cells) + 1 */
4669     NVerticesInCells = PETSC_MIN_INT;
4670     for (i = 0; i < len; i++)
4671       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4672     ++NVerticesInCells;
4673 
4674     if (numVertices == PETSC_DECIDE) numVertices = NVerticesInCells;
4675     else
4676       PetscCheck(numVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, numVertices, NVerticesInCells);
4677   }
4678   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
4679   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
4680   PetscCall(DMSetUp(dm));
4681   PetscCall(DMPlexGetCones(dm, &cones));
4682   for (c = 0; c < numCells; ++c) {
4683     for (p = 0; p < numCorners; ++p) cones[c * numCorners + p] = cells[c * numCorners + p] + numCells;
4684   }
4685   PetscCall(DMPlexSymmetrize(dm));
4686   PetscCall(DMPlexStratify(dm));
4687   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4688   PetscFunctionReturn(0);
4689 }
4690 
4691 /*@C
4692   DMPlexBuildCoordinatesFromCellList - Build DM coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4693 
4694   Input Parameters:
4695 + dm - The DM
4696 . spaceDim - The spatial dimension used for coordinates
4697 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4698 
4699   Level: advanced
4700 
4701   Notes:
4702   Not currently supported in Fortran.
4703 
4704 .seealso: `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellList()`
4705 @*/
4706 PetscErrorCode DMPlexBuildCoordinatesFromCellList(DM dm, PetscInt spaceDim, const PetscReal vertexCoords[])
4707 {
4708   PetscSection coordSection;
4709   Vec          coordinates;
4710   DM           cdm;
4711   PetscScalar *coords;
4712   PetscInt     v, vStart, vEnd, d;
4713 
4714   PetscFunctionBegin;
4715   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4716   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
4717   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4718   PetscCall(DMSetCoordinateDim(dm, spaceDim));
4719   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4720   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4721   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
4722   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
4723   for (v = vStart; v < vEnd; ++v) {
4724     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
4725     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
4726   }
4727   PetscCall(PetscSectionSetUp(coordSection));
4728 
4729   PetscCall(DMGetCoordinateDM(dm, &cdm));
4730   PetscCall(DMCreateLocalVector(cdm, &coordinates));
4731   PetscCall(VecSetBlockSize(coordinates, spaceDim));
4732   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4733   PetscCall(VecGetArrayWrite(coordinates, &coords));
4734   for (v = 0; v < vEnd - vStart; ++v) {
4735     for (d = 0; d < spaceDim; ++d) coords[v * spaceDim + d] = vertexCoords[v * spaceDim + d];
4736   }
4737   PetscCall(VecRestoreArrayWrite(coordinates, &coords));
4738   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4739   PetscCall(VecDestroy(&coordinates));
4740   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4741   PetscFunctionReturn(0);
4742 }
4743 
4744 /*@
4745   DMPlexCreateFromCellListPetsc - Create DMPLEX from a list of vertices for each cell (common mesh generator output), but only process 0 takes in the input
4746 
4747   Collective on comm
4748 
4749   Input Parameters:
4750 + comm - The communicator
4751 . dim - The topological dimension of the mesh
4752 . numCells - The number of cells, only on process 0
4753 . numVertices - The number of vertices owned by this process, or PETSC_DECIDE, only on process 0
4754 . numCorners - The number of vertices for each cell, only on process 0
4755 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4756 . cells - An array of numCells*numCorners numbers, the vertices for each cell, only on process 0
4757 . spaceDim - The spatial dimension used for coordinates
4758 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex, only on process 0
4759 
4760   Output Parameter:
4761 . dm - The DM, which only has points on process 0
4762 
4763   Notes:
4764   This function is just a convenient sequence of DMCreate(), DMSetType(), DMSetDimension(), DMPlexBuildFromCellList(),
4765   DMPlexInterpolate(), DMPlexBuildCoordinatesFromCellList()
4766 
4767   See DMPlexBuildFromCellList() for an example and details about the topology-related parameters.
4768   See DMPlexBuildCoordinatesFromCellList() for details about the geometry-related parameters.
4769   See DMPlexCreateFromCellListParallelPetsc() for parallel input
4770 
4771   Level: intermediate
4772 
4773 .seealso: `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellList()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
4774 @*/
4775 PetscErrorCode DMPlexCreateFromCellListPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], DM *dm)
4776 {
4777   PetscMPIInt rank;
4778 
4779   PetscFunctionBegin;
4780   PetscCheck(dim, comm, PETSC_ERR_ARG_OUTOFRANGE, "This is not appropriate for 0-dimensional meshes. Consider either creating the DM using DMPlexCreateFromDAG(), by hand, or using DMSwarm.");
4781   PetscCallMPI(MPI_Comm_rank(comm, &rank));
4782   PetscCall(DMCreate(comm, dm));
4783   PetscCall(DMSetType(*dm, DMPLEX));
4784   PetscCall(DMSetDimension(*dm, dim));
4785   if (rank == 0) PetscCall(DMPlexBuildFromCellList(*dm, numCells, numVertices, numCorners, cells));
4786   else PetscCall(DMPlexBuildFromCellList(*dm, 0, 0, 0, NULL));
4787   if (interpolate) {
4788     DM idm;
4789 
4790     PetscCall(DMPlexInterpolate(*dm, &idm));
4791     PetscCall(DMDestroy(dm));
4792     *dm = idm;
4793   }
4794   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, vertexCoords));
4795   else PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, NULL));
4796   PetscFunctionReturn(0);
4797 }
4798 
4799 /*@
4800   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a DM
4801 
4802   Input Parameters:
4803 + dm - The empty DM object, usually from DMCreate() and DMSetDimension()
4804 . depth - The depth of the DAG
4805 . numPoints - Array of size depth + 1 containing the number of points at each depth
4806 . coneSize - The cone size of each point
4807 . cones - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
4808 . coneOrientations - The orientation of each cone point
4809 - vertexCoords - An array of numPoints[0]*spacedim numbers representing the coordinates of each vertex, with spacedim the value set via DMSetCoordinateDim()
4810 
4811   Output Parameter:
4812 . dm - The DM
4813 
4814   Note: Two triangles sharing a face would have input
4815 $  depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
4816 $  cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
4817 $ vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
4818 $
4819 which would result in the DMPlex
4820 $
4821 $        4
4822 $      / | \
4823 $     /  |  \
4824 $    /   |   \
4825 $   2  0 | 1  5
4826 $    \   |   /
4827 $     \  |  /
4828 $      \ | /
4829 $        3
4830 $
4831 $ Notice that all points are numbered consecutively, unlike DMPlexCreateFromCellListPetsc()
4832 
4833   Level: advanced
4834 
4835 .seealso: `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
4836 @*/
4837 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
4838 {
4839   Vec          coordinates;
4840   PetscSection coordSection;
4841   PetscScalar *coords;
4842   PetscInt     coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
4843 
4844   PetscFunctionBegin;
4845   PetscCall(DMGetDimension(dm, &dim));
4846   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
4847   PetscCheck(dimEmbed >= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Embedding dimension %" PetscInt_FMT " cannot be less than intrinsic dimension %" PetscInt_FMT, dimEmbed, dim);
4848   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
4849   PetscCall(DMPlexSetChart(dm, pStart, pEnd));
4850   for (p = pStart; p < pEnd; ++p) {
4851     PetscCall(DMPlexSetConeSize(dm, p, coneSize[p - pStart]));
4852     if (firstVertex < 0 && !coneSize[p - pStart]) firstVertex = p - pStart;
4853   }
4854   PetscCheck(firstVertex >= 0 || !numPoints[0], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Expected %" PetscInt_FMT " vertices but could not find any", numPoints[0]);
4855   PetscCall(DMSetUp(dm)); /* Allocate space for cones */
4856   for (p = pStart, off = 0; p < pEnd; off += coneSize[p - pStart], ++p) {
4857     PetscCall(DMPlexSetCone(dm, p, &cones[off]));
4858     PetscCall(DMPlexSetConeOrientation(dm, p, &coneOrientations[off]));
4859   }
4860   PetscCall(DMPlexSymmetrize(dm));
4861   PetscCall(DMPlexStratify(dm));
4862   /* Build coordinates */
4863   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4864   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4865   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dimEmbed));
4866   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numPoints[0]));
4867   for (v = firstVertex; v < firstVertex + numPoints[0]; ++v) {
4868     PetscCall(PetscSectionSetDof(coordSection, v, dimEmbed));
4869     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed));
4870   }
4871   PetscCall(PetscSectionSetUp(coordSection));
4872   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4873   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
4874   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4875   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4876   PetscCall(VecSetBlockSize(coordinates, dimEmbed));
4877   PetscCall(VecSetType(coordinates, VECSTANDARD));
4878   if (vertexCoords) {
4879     PetscCall(VecGetArray(coordinates, &coords));
4880     for (v = 0; v < numPoints[0]; ++v) {
4881       PetscInt off;
4882 
4883       PetscCall(PetscSectionGetOffset(coordSection, v + firstVertex, &off));
4884       for (d = 0; d < dimEmbed; ++d) coords[off + d] = vertexCoords[v * dimEmbed + d];
4885     }
4886   }
4887   PetscCall(VecRestoreArray(coordinates, &coords));
4888   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4889   PetscCall(VecDestroy(&coordinates));
4890   PetscFunctionReturn(0);
4891 }
4892 
4893 /*@C
4894   DMPlexCreateCellVertexFromFile - Create a DMPlex mesh from a simple cell-vertex file.
4895 
4896 + comm        - The MPI communicator
4897 . filename    - Name of the .dat file
4898 - interpolate - Create faces and edges in the mesh
4899 
4900   Output Parameter:
4901 . dm  - The DM object representing the mesh
4902 
4903   Note: The format is the simplest possible:
4904 $ Ne
4905 $ v0 v1 ... vk
4906 $ Nv
4907 $ x y z marker
4908 
4909   Level: beginner
4910 
4911 .seealso: `DMPlexCreateFromFile()`, `DMPlexCreateMedFromFile()`, `DMPlexCreateGmsh()`, `DMPlexCreate()`
4912 @*/
4913 PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
4914 {
4915   DMLabel      marker;
4916   PetscViewer  viewer;
4917   Vec          coordinates;
4918   PetscSection coordSection;
4919   PetscScalar *coords;
4920   char         line[PETSC_MAX_PATH_LEN];
4921   PetscInt     dim = 3, cdim = 3, coordSize, v, c, d;
4922   PetscMPIInt  rank;
4923   int          snum, Nv, Nc, Ncn, Nl;
4924 
4925   PetscFunctionBegin;
4926   PetscCallMPI(MPI_Comm_rank(comm, &rank));
4927   PetscCall(PetscViewerCreate(comm, &viewer));
4928   PetscCall(PetscViewerSetType(viewer, PETSCVIEWERASCII));
4929   PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
4930   PetscCall(PetscViewerFileSetName(viewer, filename));
4931   if (rank == 0) {
4932     PetscCall(PetscViewerRead(viewer, line, 4, NULL, PETSC_STRING));
4933     snum = sscanf(line, "%d %d %d %d", &Nc, &Nv, &Ncn, &Nl);
4934     PetscCheck(snum == 4, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4935   } else {
4936     Nc = Nv = Ncn = Nl = 0;
4937   }
4938   PetscCall(DMCreate(comm, dm));
4939   PetscCall(DMSetType(*dm, DMPLEX));
4940   PetscCall(DMPlexSetChart(*dm, 0, Nc + Nv));
4941   PetscCall(DMSetDimension(*dm, dim));
4942   PetscCall(DMSetCoordinateDim(*dm, cdim));
4943   /* Read topology */
4944   if (rank == 0) {
4945     char     format[PETSC_MAX_PATH_LEN];
4946     PetscInt cone[8];
4947     int      vbuf[8], v;
4948 
4949     for (c = 0; c < Ncn; ++c) {
4950       format[c * 3 + 0] = '%';
4951       format[c * 3 + 1] = 'd';
4952       format[c * 3 + 2] = ' ';
4953     }
4954     format[Ncn * 3 - 1] = '\0';
4955     for (c = 0; c < Nc; ++c) PetscCall(DMPlexSetConeSize(*dm, c, Ncn));
4956     PetscCall(DMSetUp(*dm));
4957     for (c = 0; c < Nc; ++c) {
4958       PetscCall(PetscViewerRead(viewer, line, Ncn, NULL, PETSC_STRING));
4959       switch (Ncn) {
4960       case 2:
4961         snum = sscanf(line, format, &vbuf[0], &vbuf[1]);
4962         break;
4963       case 3:
4964         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2]);
4965         break;
4966       case 4:
4967         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3]);
4968         break;
4969       case 6:
4970         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5]);
4971         break;
4972       case 8:
4973         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);
4974         break;
4975       default:
4976         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No cell shape with %d vertices", Ncn);
4977       }
4978       PetscCheck(snum == Ncn, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4979       for (v = 0; v < Ncn; ++v) cone[v] = vbuf[v] + Nc;
4980       /* Hexahedra are inverted */
4981       if (Ncn == 8) {
4982         PetscInt tmp = cone[1];
4983         cone[1]      = cone[3];
4984         cone[3]      = tmp;
4985       }
4986       PetscCall(DMPlexSetCone(*dm, c, cone));
4987     }
4988   }
4989   PetscCall(DMPlexSymmetrize(*dm));
4990   PetscCall(DMPlexStratify(*dm));
4991   /* Read coordinates */
4992   PetscCall(DMGetCoordinateSection(*dm, &coordSection));
4993   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4994   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
4995   PetscCall(PetscSectionSetChart(coordSection, Nc, Nc + Nv));
4996   for (v = Nc; v < Nc + Nv; ++v) {
4997     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
4998     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
4999   }
5000   PetscCall(PetscSectionSetUp(coordSection));
5001   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
5002   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
5003   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5004   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
5005   PetscCall(VecSetBlockSize(coordinates, cdim));
5006   PetscCall(VecSetType(coordinates, VECSTANDARD));
5007   PetscCall(VecGetArray(coordinates, &coords));
5008   if (rank == 0) {
5009     char   format[PETSC_MAX_PATH_LEN];
5010     double x[3];
5011     int    l, val[3];
5012 
5013     if (Nl) {
5014       for (l = 0; l < Nl; ++l) {
5015         format[l * 3 + 0] = '%';
5016         format[l * 3 + 1] = 'd';
5017         format[l * 3 + 2] = ' ';
5018       }
5019       format[Nl * 3 - 1] = '\0';
5020       PetscCall(DMCreateLabel(*dm, "marker"));
5021       PetscCall(DMGetLabel(*dm, "marker", &marker));
5022     }
5023     for (v = 0; v < Nv; ++v) {
5024       PetscCall(PetscViewerRead(viewer, line, 3 + Nl, NULL, PETSC_STRING));
5025       snum = sscanf(line, "%lg %lg %lg", &x[0], &x[1], &x[2]);
5026       PetscCheck(snum == 3, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5027       switch (Nl) {
5028       case 0:
5029         snum = 0;
5030         break;
5031       case 1:
5032         snum = sscanf(line, format, &val[0]);
5033         break;
5034       case 2:
5035         snum = sscanf(line, format, &val[0], &val[1]);
5036         break;
5037       case 3:
5038         snum = sscanf(line, format, &val[0], &val[1], &val[2]);
5039         break;
5040       default:
5041         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Request support for %d labels", Nl);
5042       }
5043       PetscCheck(snum == Nl, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5044       for (d = 0; d < cdim; ++d) coords[v * cdim + d] = x[d];
5045       for (l = 0; l < Nl; ++l) PetscCall(DMLabelSetValue(marker, v + Nc, val[l]));
5046     }
5047   }
5048   PetscCall(VecRestoreArray(coordinates, &coords));
5049   PetscCall(DMSetCoordinatesLocal(*dm, coordinates));
5050   PetscCall(VecDestroy(&coordinates));
5051   PetscCall(PetscViewerDestroy(&viewer));
5052   if (interpolate) {
5053     DM      idm;
5054     DMLabel bdlabel;
5055 
5056     PetscCall(DMPlexInterpolate(*dm, &idm));
5057     PetscCall(DMDestroy(dm));
5058     *dm = idm;
5059 
5060     if (!Nl) {
5061       PetscCall(DMCreateLabel(*dm, "marker"));
5062       PetscCall(DMGetLabel(*dm, "marker", &bdlabel));
5063       PetscCall(DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel));
5064       PetscCall(DMPlexLabelComplete(*dm, bdlabel));
5065     }
5066   }
5067   PetscFunctionReturn(0);
5068 }
5069 
5070 /*@C
5071   DMPlexCreateFromFile - This takes a filename and produces a DM
5072 
5073   Input Parameters:
5074 + comm - The communicator
5075 . filename - A file name
5076 . plexname - The object name of the resulting DM, also used for intra-datafile lookup by some formats
5077 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
5078 
5079   Output Parameter:
5080 . dm - The DM
5081 
5082   Options Database Keys:
5083 . -dm_plex_create_from_hdf5_xdmf - use the PETSC_VIEWER_HDF5_XDMF format for reading HDF5
5084 
5085   Use -dm_plex_create_ prefix to pass options to the internal PetscViewer, e.g.
5086 $ -dm_plex_create_viewer_hdf5_collective
5087 
5088   Notes:
5089   Using PETSCVIEWERHDF5 type with PETSC_VIEWER_HDF5_PETSC format, one can save multiple DMPlex
5090   meshes in a single HDF5 file. This in turn requires one to name the DMPlex object with PetscObjectSetName()
5091   before saving it with DMView() and before loading it with DMLoad() for identification of the mesh object.
5092   The input parameter name is thus used to name the DMPlex object when DMPlexCreateFromFile() internally
5093   calls DMLoad(). Currently, name is ignored for other viewer types and/or formats.
5094 
5095   Level: beginner
5096 
5097 .seealso: `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`, `PetscObjectSetName()`, `DMView()`, `DMLoad()`
5098 @*/
5099 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], const char plexname[], PetscBool interpolate, DM *dm)
5100 {
5101   const char  extGmsh[]      = ".msh";
5102   const char  extGmsh2[]     = ".msh2";
5103   const char  extGmsh4[]     = ".msh4";
5104   const char  extCGNS[]      = ".cgns";
5105   const char  extExodus[]    = ".exo";
5106   const char  extExodus_e[]  = ".e";
5107   const char  extGenesis[]   = ".gen";
5108   const char  extFluent[]    = ".cas";
5109   const char  extHDF5[]      = ".h5";
5110   const char  extMed[]       = ".med";
5111   const char  extPLY[]       = ".ply";
5112   const char  extEGADSLite[] = ".egadslite";
5113   const char  extEGADS[]     = ".egads";
5114   const char  extIGES[]      = ".igs";
5115   const char  extSTEP[]      = ".stp";
5116   const char  extCV[]        = ".dat";
5117   size_t      len;
5118   PetscBool   isGmsh, isGmsh2, isGmsh4, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isMed, isPLY, isEGADSLite, isEGADS, isIGES, isSTEP, isCV;
5119   PetscMPIInt rank;
5120 
5121   PetscFunctionBegin;
5122   PetscValidCharPointer(filename, 2);
5123   if (plexname) PetscValidCharPointer(plexname, 3);
5124   PetscValidPointer(dm, 5);
5125   PetscCall(DMInitializePackage());
5126   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5127   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5128   PetscCall(PetscStrlen(filename, &len));
5129   PetscCheck(len, comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
5130 
5131 #define CheckExtension(extension__, is_extension__) \
5132   do { \
5133     PetscAssert(sizeof(extension__), comm, PETSC_ERR_PLIB, "Zero-size extension: %s", extension__); \
5134     /* don't count the null-terminator at the end */ \
5135     const size_t ext_len = sizeof(extension__) - 1; \
5136     if (len < ext_len) { \
5137       is_extension__ = PETSC_FALSE; \
5138     } else { \
5139       PetscCall(PetscStrncmp(filename + len - ext_len, extension__, ext_len, &is_extension__)); \
5140     } \
5141   } while (0)
5142 
5143   CheckExtension(extGmsh, isGmsh);
5144   CheckExtension(extGmsh2, isGmsh2);
5145   CheckExtension(extGmsh4, isGmsh4);
5146   CheckExtension(extCGNS, isCGNS);
5147   CheckExtension(extExodus, isExodus);
5148   if (!isExodus) CheckExtension(extExodus_e, isExodus);
5149   CheckExtension(extGenesis, isGenesis);
5150   CheckExtension(extFluent, isFluent);
5151   CheckExtension(extHDF5, isHDF5);
5152   CheckExtension(extMed, isMed);
5153   CheckExtension(extPLY, isPLY);
5154   CheckExtension(extEGADSLite, isEGADSLite);
5155   CheckExtension(extEGADS, isEGADS);
5156   CheckExtension(extIGES, isIGES);
5157   CheckExtension(extSTEP, isSTEP);
5158   CheckExtension(extCV, isCV);
5159 
5160 #undef CheckExtension
5161 
5162   if (isGmsh || isGmsh2 || isGmsh4) {
5163     PetscCall(DMPlexCreateGmshFromFile(comm, filename, interpolate, dm));
5164   } else if (isCGNS) {
5165     PetscCall(DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm));
5166   } else if (isExodus || isGenesis) {
5167     PetscCall(DMPlexCreateExodusFromFile(comm, filename, interpolate, dm));
5168   } else if (isFluent) {
5169     PetscCall(DMPlexCreateFluentFromFile(comm, filename, interpolate, dm));
5170   } else if (isHDF5) {
5171     PetscBool   load_hdf5_xdmf = PETSC_FALSE;
5172     PetscViewer viewer;
5173 
5174     /* PETSC_VIEWER_HDF5_XDMF is used if the filename ends with .xdmf.h5, or if -dm_plex_create_from_hdf5_xdmf option is present */
5175     PetscCall(PetscStrncmp(&filename[PetscMax(0, len - 8)], ".xdmf", 5, &load_hdf5_xdmf));
5176     PetscCall(PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &load_hdf5_xdmf, NULL));
5177     PetscCall(PetscViewerCreate(comm, &viewer));
5178     PetscCall(PetscViewerSetType(viewer, PETSCVIEWERHDF5));
5179     PetscCall(PetscViewerSetOptionsPrefix(viewer, "dm_plex_create_"));
5180     PetscCall(PetscViewerSetFromOptions(viewer));
5181     PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
5182     PetscCall(PetscViewerFileSetName(viewer, filename));
5183 
5184     PetscCall(DMCreate(comm, dm));
5185     PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5186     PetscCall(DMSetType(*dm, DMPLEX));
5187     if (load_hdf5_xdmf) PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF));
5188     PetscCall(DMLoad(*dm, viewer));
5189     if (load_hdf5_xdmf) PetscCall(PetscViewerPopFormat(viewer));
5190     PetscCall(PetscViewerDestroy(&viewer));
5191 
5192     if (interpolate) {
5193       DM idm;
5194 
5195       PetscCall(DMPlexInterpolate(*dm, &idm));
5196       PetscCall(DMDestroy(dm));
5197       *dm = idm;
5198     }
5199   } else if (isMed) {
5200     PetscCall(DMPlexCreateMedFromFile(comm, filename, interpolate, dm));
5201   } else if (isPLY) {
5202     PetscCall(DMPlexCreatePLYFromFile(comm, filename, interpolate, dm));
5203   } else if (isEGADSLite || isEGADS || isIGES || isSTEP) {
5204     if (isEGADSLite) PetscCall(DMPlexCreateEGADSLiteFromFile(comm, filename, dm));
5205     else PetscCall(DMPlexCreateEGADSFromFile(comm, filename, dm));
5206     if (!interpolate) {
5207       DM udm;
5208 
5209       PetscCall(DMPlexUninterpolate(*dm, &udm));
5210       PetscCall(DMDestroy(dm));
5211       *dm = udm;
5212     }
5213   } else if (isCV) {
5214     PetscCall(DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm));
5215   } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
5216   PetscCall(PetscStrlen(plexname, &len));
5217   if (len) PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5218   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5219   PetscFunctionReturn(0);
5220 }
5221