xref: /petsc/src/dm/impls/plex/plexcreate.c (revision 5e17fc226d2046beb8e6d40a3336e33829fdc69f)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h> /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>   /*I   "petscdmplex.h"   I*/
4 #include <petscsf.h>
5 #include <petscdmplextransform.h>
6 #include <petscdmlabelephemeral.h>
7 #include <petsc/private/kernels/blockmatmult.h>
8 #include <petsc/private/kernels/blockinvert.h>
9 
10 PetscLogEvent DMPLEX_CreateFromFile, DMPLEX_BuildFromCellList, DMPLEX_BuildCoordinatesFromCellList;
11 
12 /* External function declarations here */
13 static PetscErrorCode DMInitialize_Plex(DM dm);
14 
15 /* This copies internal things in the Plex structure that we generally want when making a new, related Plex */
16 PetscErrorCode DMPlexCopy_Internal(DM dmin, PetscBool copyPeriodicity, PetscBool copyOverlap, DM dmout)
17 {
18   const PetscReal         *maxCell, *Lstart, *L;
19   PetscBool                dist;
20   DMPlexReorderDefaultFlag reorder;
21 
22   PetscFunctionBegin;
23   if (copyPeriodicity) {
24     PetscCall(DMGetPeriodicity(dmin, &maxCell, &Lstart, &L));
25     PetscCall(DMSetPeriodicity(dmout, maxCell, Lstart, L));
26   }
27   PetscCall(DMPlexDistributeGetDefault(dmin, &dist));
28   PetscCall(DMPlexDistributeSetDefault(dmout, dist));
29   PetscCall(DMPlexReorderGetDefault(dmin, &reorder));
30   PetscCall(DMPlexReorderSetDefault(dmout, reorder));
31   ((DM_Plex *)dmout->data)->useHashLocation = ((DM_Plex *)dmin->data)->useHashLocation;
32   if (copyOverlap) PetscCall(DMPlexSetOverlap_Plex(dmout, dmin, 0));
33   PetscFunctionReturn(0);
34 }
35 
36 /* Replace dm with the contents of ndm, and then destroy ndm
37    - Share the DM_Plex structure
38    - Share the coordinates
39    - Share the SF
40 */
41 PetscErrorCode DMPlexReplace_Internal(DM dm, DM *ndm)
42 {
43   PetscSF          sf;
44   DM               dmNew = *ndm, coordDM, coarseDM;
45   Vec              coords;
46   const PetscReal *maxCell, *Lstart, *L;
47   PetscInt         dim, cdim;
48 
49   PetscFunctionBegin;
50   if (dm == dmNew) {
51     PetscCall(DMDestroy(ndm));
52     PetscFunctionReturn(0);
53   }
54   dm->setupcalled = dmNew->setupcalled;
55   PetscCall(DMGetDimension(dmNew, &dim));
56   PetscCall(DMSetDimension(dm, dim));
57   PetscCall(DMGetCoordinateDim(dmNew, &cdim));
58   PetscCall(DMSetCoordinateDim(dm, cdim));
59   PetscCall(DMGetPointSF(dmNew, &sf));
60   PetscCall(DMSetPointSF(dm, sf));
61   PetscCall(DMGetCoordinateDM(dmNew, &coordDM));
62   PetscCall(DMGetCoordinatesLocal(dmNew, &coords));
63   PetscCall(DMSetCoordinateDM(dm, coordDM));
64   PetscCall(DMSetCoordinatesLocal(dm, coords));
65   PetscCall(DMGetCellCoordinateDM(dmNew, &coordDM));
66   PetscCall(DMGetCellCoordinatesLocal(dmNew, &coords));
67   PetscCall(DMSetCellCoordinateDM(dm, coordDM));
68   PetscCall(DMSetCellCoordinatesLocal(dm, coords));
69   /* Do not want to create the coordinate field if it does not already exist, so do not call DMGetCoordinateField() */
70   PetscCall(DMFieldDestroy(&dm->coordinates[0].field));
71   dm->coordinates[0].field            = dmNew->coordinates[0].field;
72   ((DM_Plex *)dmNew->data)->coordFunc = ((DM_Plex *)dm->data)->coordFunc;
73   PetscCall(DMGetPeriodicity(dmNew, &maxCell, &Lstart, &L));
74   PetscCall(DMSetPeriodicity(dm, maxCell, Lstart, L));
75   PetscCall(DMDestroy_Plex(dm));
76   PetscCall(DMInitialize_Plex(dm));
77   dm->data = dmNew->data;
78   ((DM_Plex *)dmNew->data)->refct++;
79   PetscCall(DMDestroyLabelLinkList_Internal(dm));
80   PetscCall(DMCopyLabels(dmNew, dm, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL));
81   PetscCall(DMGetCoarseDM(dmNew, &coarseDM));
82   PetscCall(DMSetCoarseDM(dm, coarseDM));
83   PetscCall(DMDestroy(ndm));
84   PetscFunctionReturn(0);
85 }
86 
87 /* Swap dm with the contents of dmNew
88    - Swap the DM_Plex structure
89    - Swap the coordinates
90    - Swap the point PetscSF
91 */
92 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
93 {
94   DM          coordDMA, coordDMB;
95   Vec         coordsA, coordsB;
96   PetscSF     sfA, sfB;
97   DMField     fieldTmp;
98   void       *tmp;
99   DMLabelLink listTmp;
100   DMLabel     depthTmp;
101   PetscInt    tmpI;
102 
103   PetscFunctionBegin;
104   if (dmA == dmB) PetscFunctionReturn(0);
105   PetscCall(DMGetPointSF(dmA, &sfA));
106   PetscCall(DMGetPointSF(dmB, &sfB));
107   PetscCall(PetscObjectReference((PetscObject)sfA));
108   PetscCall(DMSetPointSF(dmA, sfB));
109   PetscCall(DMSetPointSF(dmB, sfA));
110   PetscCall(PetscObjectDereference((PetscObject)sfA));
111 
112   PetscCall(DMGetCoordinateDM(dmA, &coordDMA));
113   PetscCall(DMGetCoordinateDM(dmB, &coordDMB));
114   PetscCall(PetscObjectReference((PetscObject)coordDMA));
115   PetscCall(DMSetCoordinateDM(dmA, coordDMB));
116   PetscCall(DMSetCoordinateDM(dmB, coordDMA));
117   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
118 
119   PetscCall(DMGetCoordinatesLocal(dmA, &coordsA));
120   PetscCall(DMGetCoordinatesLocal(dmB, &coordsB));
121   PetscCall(PetscObjectReference((PetscObject)coordsA));
122   PetscCall(DMSetCoordinatesLocal(dmA, coordsB));
123   PetscCall(DMSetCoordinatesLocal(dmB, coordsA));
124   PetscCall(PetscObjectDereference((PetscObject)coordsA));
125 
126   PetscCall(DMGetCellCoordinateDM(dmA, &coordDMA));
127   PetscCall(DMGetCellCoordinateDM(dmB, &coordDMB));
128   PetscCall(PetscObjectReference((PetscObject)coordDMA));
129   PetscCall(DMSetCellCoordinateDM(dmA, coordDMB));
130   PetscCall(DMSetCellCoordinateDM(dmB, coordDMA));
131   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
132 
133   PetscCall(DMGetCellCoordinatesLocal(dmA, &coordsA));
134   PetscCall(DMGetCellCoordinatesLocal(dmB, &coordsB));
135   PetscCall(PetscObjectReference((PetscObject)coordsA));
136   PetscCall(DMSetCellCoordinatesLocal(dmA, coordsB));
137   PetscCall(DMSetCellCoordinatesLocal(dmB, coordsA));
138   PetscCall(PetscObjectDereference((PetscObject)coordsA));
139 
140   fieldTmp                  = dmA->coordinates[0].field;
141   dmA->coordinates[0].field = dmB->coordinates[0].field;
142   dmB->coordinates[0].field = fieldTmp;
143   fieldTmp                  = dmA->coordinates[1].field;
144   dmA->coordinates[1].field = dmB->coordinates[1].field;
145   dmB->coordinates[1].field = fieldTmp;
146   tmp                       = dmA->data;
147   dmA->data                 = dmB->data;
148   dmB->data                 = tmp;
149   listTmp                   = dmA->labels;
150   dmA->labels               = dmB->labels;
151   dmB->labels               = listTmp;
152   depthTmp                  = dmA->depthLabel;
153   dmA->depthLabel           = dmB->depthLabel;
154   dmB->depthLabel           = depthTmp;
155   depthTmp                  = dmA->celltypeLabel;
156   dmA->celltypeLabel        = dmB->celltypeLabel;
157   dmB->celltypeLabel        = depthTmp;
158   tmpI                      = dmA->levelup;
159   dmA->levelup              = dmB->levelup;
160   dmB->levelup              = tmpI;
161   PetscFunctionReturn(0);
162 }
163 
164 static PetscErrorCode DMPlexInterpolateInPlace_Internal(DM dm)
165 {
166   DM idm;
167 
168   PetscFunctionBegin;
169   PetscCall(DMPlexInterpolate(dm, &idm));
170   PetscCall(DMPlexCopyCoordinates(dm, idm));
171   PetscCall(DMPlexReplace_Internal(dm, &idm));
172   PetscFunctionReturn(0);
173 }
174 
175 /*@C
176   DMPlexCreateCoordinateSpace - Creates a finite element space for the coordinates
177 
178   Collective
179 
180   Input Parameters:
181 + DM        - The DM
182 . degree    - The degree of the finite element or PETSC_DECIDE
183 - coordFunc - An optional function to map new points from refinement to the surface
184 
185   Level: advanced
186 
187 .seealso: `PetscFECreateLagrange()`, `DMGetCoordinateDM()`
188 @*/
189 PetscErrorCode DMPlexCreateCoordinateSpace(DM dm, PetscInt degree, PetscPointFunc coordFunc)
190 {
191   DM_Plex     *mesh = (DM_Plex *)dm->data;
192   DM           cdm;
193   PetscDS      cds;
194   PetscFE      fe;
195   PetscClassId id;
196 
197   PetscFunctionBegin;
198   PetscCall(DMGetCoordinateDM(dm, &cdm));
199   PetscCall(DMGetDS(cdm, &cds));
200   PetscCall(PetscDSGetDiscretization(cds, 0, (PetscObject *)&fe));
201   PetscCall(PetscObjectGetClassId((PetscObject)fe, &id));
202   if (id != PETSCFE_CLASSID) {
203     PetscBool simplex;
204     PetscInt  dim, dE, qorder;
205 
206     PetscCall(DMGetDimension(dm, &dim));
207     PetscCall(DMGetCoordinateDim(dm, &dE));
208     qorder = degree;
209     PetscObjectOptionsBegin((PetscObject)cdm);
210     PetscCall(PetscOptionsBoundedInt("-coord_dm_default_quadrature_order", "Quadrature order is one less than quadrature points per edge", "DMPlexCreateCoordinateSpace", qorder, &qorder, NULL, 0));
211     PetscOptionsEnd();
212     if (degree == PETSC_DECIDE) fe = NULL;
213     else {
214       PetscCall(DMPlexIsSimplex(dm, &simplex));
215       PetscCall(PetscFECreateLagrange(PETSC_COMM_SELF, dim, dE, simplex, degree, qorder, &fe));
216     }
217     PetscCall(DMProjectCoordinates(dm, fe));
218     PetscCall(PetscFEDestroy(&fe));
219   }
220   mesh->coordFunc = coordFunc;
221   PetscFunctionReturn(0);
222 }
223 
224 /*@
225   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
226 
227   Collective
228 
229   Input Parameters:
230 + comm - The communicator for the DM object
231 . dim - The spatial dimension
232 . simplex - Flag for simplicial cells, otherwise they are tensor product cells
233 . interpolate - Flag to create intermediate mesh pieces (edges, faces)
234 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
235 
236   Output Parameter:
237 . dm - The DM object
238 
239   Level: beginner
240 
241 .seealso: `DMSetType()`, `DMCreate()`
242 @*/
243 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscReal refinementLimit, DM *newdm)
244 {
245   DM          dm;
246   PetscMPIInt rank;
247 
248   PetscFunctionBegin;
249   PetscCall(DMCreate(comm, &dm));
250   PetscCall(DMSetType(dm, DMPLEX));
251   PetscCall(DMSetDimension(dm, dim));
252   PetscCallMPI(MPI_Comm_rank(comm, &rank));
253   switch (dim) {
254   case 2:
255     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "triangular"));
256     else PetscCall(PetscObjectSetName((PetscObject)dm, "quadrilateral"));
257     break;
258   case 3:
259     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "tetrahedral"));
260     else PetscCall(PetscObjectSetName((PetscObject)dm, "hexahedral"));
261     break;
262   default:
263     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
264   }
265   if (rank) {
266     PetscInt numPoints[2] = {0, 0};
267     PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL));
268   } else {
269     switch (dim) {
270     case 2:
271       if (simplex) {
272         PetscInt    numPoints[2]        = {4, 2};
273         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
274         PetscInt    cones[6]            = {2, 3, 4, 5, 4, 3};
275         PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
276         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
277 
278         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
279       } else {
280         PetscInt    numPoints[2]        = {6, 2};
281         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
282         PetscInt    cones[8]            = {2, 3, 4, 5, 3, 6, 7, 4};
283         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
284         PetscScalar vertexCoords[12]    = {-1.0, -0.5, 0.0, -0.5, 0.0, 0.5, -1.0, 0.5, 1.0, -0.5, 1.0, 0.5};
285 
286         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
287       }
288       break;
289     case 3:
290       if (simplex) {
291         PetscInt    numPoints[2]        = {5, 2};
292         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
293         PetscInt    cones[8]            = {4, 3, 5, 2, 5, 3, 4, 6};
294         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
295         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0};
296 
297         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
298       } else {
299         PetscInt    numPoints[2]         = {12, 2};
300         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
301         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9, 5, 4, 10, 11, 7, 12, 13, 8};
302         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
303         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5, -1.0, 0.5, -0.5, 0.0, 0.5, -0.5, 0.0, -0.5, -0.5, -1.0, -0.5, 0.5, 0.0, -0.5, 0.5, 0.0, 0.5, 0.5, -1.0, 0.5, 0.5, 1.0, 0.5, -0.5, 1.0, -0.5, -0.5, 1.0, -0.5, 0.5, 1.0, 0.5, 0.5};
304 
305         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
306       }
307       break;
308     default:
309       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
310     }
311   }
312   *newdm = dm;
313   if (refinementLimit > 0.0) {
314     DM          rdm;
315     const char *name;
316 
317     PetscCall(DMPlexSetRefinementUniform(*newdm, PETSC_FALSE));
318     PetscCall(DMPlexSetRefinementLimit(*newdm, refinementLimit));
319     PetscCall(DMRefine(*newdm, comm, &rdm));
320     PetscCall(PetscObjectGetName((PetscObject)*newdm, &name));
321     PetscCall(PetscObjectSetName((PetscObject)rdm, name));
322     PetscCall(DMDestroy(newdm));
323     *newdm = rdm;
324   }
325   if (interpolate) {
326     DM idm;
327 
328     PetscCall(DMPlexInterpolate(*newdm, &idm));
329     PetscCall(DMDestroy(newdm));
330     *newdm = idm;
331   }
332   PetscFunctionReturn(0);
333 }
334 
335 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
336 {
337   const PetscInt numVertices    = 2;
338   PetscInt       markerRight    = 1;
339   PetscInt       markerLeft     = 1;
340   PetscBool      markerSeparate = PETSC_FALSE;
341   Vec            coordinates;
342   PetscSection   coordSection;
343   PetscScalar   *coords;
344   PetscInt       coordSize;
345   PetscMPIInt    rank;
346   PetscInt       cdim = 1, v;
347 
348   PetscFunctionBegin;
349   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
350   if (markerSeparate) {
351     markerRight = 2;
352     markerLeft  = 1;
353   }
354   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
355   if (rank == 0) {
356     PetscCall(DMPlexSetChart(dm, 0, numVertices));
357     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
358     PetscCall(DMSetLabelValue(dm, "marker", 0, markerLeft));
359     PetscCall(DMSetLabelValue(dm, "marker", 1, markerRight));
360   }
361   PetscCall(DMPlexSymmetrize(dm));
362   PetscCall(DMPlexStratify(dm));
363   /* Build coordinates */
364   PetscCall(DMSetCoordinateDim(dm, cdim));
365   PetscCall(DMGetCoordinateSection(dm, &coordSection));
366   PetscCall(PetscSectionSetNumFields(coordSection, 1));
367   PetscCall(PetscSectionSetChart(coordSection, 0, numVertices));
368   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
369   for (v = 0; v < numVertices; ++v) {
370     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
371     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
372   }
373   PetscCall(PetscSectionSetUp(coordSection));
374   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
375   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
376   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
377   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
378   PetscCall(VecSetBlockSize(coordinates, cdim));
379   PetscCall(VecSetType(coordinates, VECSTANDARD));
380   PetscCall(VecGetArray(coordinates, &coords));
381   coords[0] = lower[0];
382   coords[1] = upper[0];
383   PetscCall(VecRestoreArray(coordinates, &coords));
384   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
385   PetscCall(VecDestroy(&coordinates));
386   PetscFunctionReturn(0);
387 }
388 
389 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
390 {
391   const PetscInt numVertices    = (edges[0] + 1) * (edges[1] + 1);
392   const PetscInt numEdges       = edges[0] * (edges[1] + 1) + (edges[0] + 1) * edges[1];
393   PetscInt       markerTop      = 1;
394   PetscInt       markerBottom   = 1;
395   PetscInt       markerRight    = 1;
396   PetscInt       markerLeft     = 1;
397   PetscBool      markerSeparate = PETSC_FALSE;
398   Vec            coordinates;
399   PetscSection   coordSection;
400   PetscScalar   *coords;
401   PetscInt       coordSize;
402   PetscMPIInt    rank;
403   PetscInt       v, vx, vy;
404 
405   PetscFunctionBegin;
406   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
407   if (markerSeparate) {
408     markerTop    = 3;
409     markerBottom = 1;
410     markerRight  = 2;
411     markerLeft   = 4;
412   }
413   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
414   if (rank == 0) {
415     PetscInt e, ex, ey;
416 
417     PetscCall(DMPlexSetChart(dm, 0, numEdges + numVertices));
418     for (e = 0; e < numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
419     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
420     for (vx = 0; vx <= edges[0]; vx++) {
421       for (ey = 0; ey < edges[1]; ey++) {
422         PetscInt edge   = vx * edges[1] + ey + edges[0] * (edges[1] + 1);
423         PetscInt vertex = ey * (edges[0] + 1) + vx + numEdges;
424         PetscInt cone[2];
425 
426         cone[0] = vertex;
427         cone[1] = vertex + edges[0] + 1;
428         PetscCall(DMPlexSetCone(dm, edge, cone));
429         if (vx == edges[0]) {
430           PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
431           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
432           if (ey == edges[1] - 1) {
433             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
434             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerRight));
435           }
436         } else if (vx == 0) {
437           PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
438           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
439           if (ey == edges[1] - 1) {
440             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
441             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft));
442           }
443         }
444       }
445     }
446     for (vy = 0; vy <= edges[1]; vy++) {
447       for (ex = 0; ex < edges[0]; ex++) {
448         PetscInt edge   = vy * edges[0] + ex;
449         PetscInt vertex = vy * (edges[0] + 1) + ex + numEdges;
450         PetscInt cone[2];
451 
452         cone[0] = vertex;
453         cone[1] = vertex + 1;
454         PetscCall(DMPlexSetCone(dm, edge, cone));
455         if (vy == edges[1]) {
456           PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
457           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
458           if (ex == edges[0] - 1) {
459             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
460             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerTop));
461           }
462         } else if (vy == 0) {
463           PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
464           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
465           if (ex == edges[0] - 1) {
466             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
467             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom));
468           }
469         }
470       }
471     }
472   }
473   PetscCall(DMPlexSymmetrize(dm));
474   PetscCall(DMPlexStratify(dm));
475   /* Build coordinates */
476   PetscCall(DMSetCoordinateDim(dm, 2));
477   PetscCall(DMGetCoordinateSection(dm, &coordSection));
478   PetscCall(PetscSectionSetNumFields(coordSection, 1));
479   PetscCall(PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices));
480   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 2));
481   for (v = numEdges; v < numEdges + numVertices; ++v) {
482     PetscCall(PetscSectionSetDof(coordSection, v, 2));
483     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 2));
484   }
485   PetscCall(PetscSectionSetUp(coordSection));
486   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
487   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
488   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
489   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
490   PetscCall(VecSetBlockSize(coordinates, 2));
491   PetscCall(VecSetType(coordinates, VECSTANDARD));
492   PetscCall(VecGetArray(coordinates, &coords));
493   for (vy = 0; vy <= edges[1]; ++vy) {
494     for (vx = 0; vx <= edges[0]; ++vx) {
495       coords[(vy * (edges[0] + 1) + vx) * 2 + 0] = lower[0] + ((upper[0] - lower[0]) / edges[0]) * vx;
496       coords[(vy * (edges[0] + 1) + vx) * 2 + 1] = lower[1] + ((upper[1] - lower[1]) / edges[1]) * vy;
497     }
498   }
499   PetscCall(VecRestoreArray(coordinates, &coords));
500   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
501   PetscCall(VecDestroy(&coordinates));
502   PetscFunctionReturn(0);
503 }
504 
505 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
506 {
507   PetscInt     vertices[3], numVertices;
508   PetscInt     numFaces       = 2 * faces[0] * faces[1] + 2 * faces[1] * faces[2] + 2 * faces[0] * faces[2];
509   PetscInt     markerTop      = 1;
510   PetscInt     markerBottom   = 1;
511   PetscInt     markerFront    = 1;
512   PetscInt     markerBack     = 1;
513   PetscInt     markerRight    = 1;
514   PetscInt     markerLeft     = 1;
515   PetscBool    markerSeparate = PETSC_FALSE;
516   Vec          coordinates;
517   PetscSection coordSection;
518   PetscScalar *coords;
519   PetscInt     coordSize;
520   PetscMPIInt  rank;
521   PetscInt     v, vx, vy, vz;
522   PetscInt     voffset, iface = 0, cone[4];
523 
524   PetscFunctionBegin;
525   PetscCheck(faces[0] >= 1 && faces[1] >= 1 && faces[2] >= 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
526   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
527   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
528   if (markerSeparate) {
529     markerBottom = 1;
530     markerTop    = 2;
531     markerFront  = 3;
532     markerBack   = 4;
533     markerRight  = 5;
534     markerLeft   = 6;
535   }
536   vertices[0] = faces[0] + 1;
537   vertices[1] = faces[1] + 1;
538   vertices[2] = faces[2] + 1;
539   numVertices = vertices[0] * vertices[1] * vertices[2];
540   if (rank == 0) {
541     PetscInt f;
542 
543     PetscCall(DMPlexSetChart(dm, 0, numFaces + numVertices));
544     for (f = 0; f < numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
545     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
546 
547     /* Side 0 (Top) */
548     for (vy = 0; vy < faces[1]; vy++) {
549       for (vx = 0; vx < faces[0]; vx++) {
550         voffset = numFaces + vertices[0] * vertices[1] * (vertices[2] - 1) + vy * vertices[0] + vx;
551         cone[0] = voffset;
552         cone[1] = voffset + 1;
553         cone[2] = voffset + vertices[0] + 1;
554         cone[3] = voffset + vertices[0];
555         PetscCall(DMPlexSetCone(dm, iface, cone));
556         PetscCall(DMSetLabelValue(dm, "marker", iface, markerTop));
557         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerTop));
558         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerTop));
559         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerTop));
560         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerTop));
561         iface++;
562       }
563     }
564 
565     /* Side 1 (Bottom) */
566     for (vy = 0; vy < faces[1]; vy++) {
567       for (vx = 0; vx < faces[0]; vx++) {
568         voffset = numFaces + vy * (faces[0] + 1) + vx;
569         cone[0] = voffset + 1;
570         cone[1] = voffset;
571         cone[2] = voffset + vertices[0];
572         cone[3] = voffset + vertices[0] + 1;
573         PetscCall(DMPlexSetCone(dm, iface, cone));
574         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBottom));
575         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBottom));
576         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBottom));
577         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerBottom));
578         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerBottom));
579         iface++;
580       }
581     }
582 
583     /* Side 2 (Front) */
584     for (vz = 0; vz < faces[2]; vz++) {
585       for (vx = 0; vx < faces[0]; vx++) {
586         voffset = numFaces + vz * vertices[0] * vertices[1] + vx;
587         cone[0] = voffset;
588         cone[1] = voffset + 1;
589         cone[2] = voffset + vertices[0] * vertices[1] + 1;
590         cone[3] = voffset + vertices[0] * vertices[1];
591         PetscCall(DMPlexSetCone(dm, iface, cone));
592         PetscCall(DMSetLabelValue(dm, "marker", iface, markerFront));
593         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerFront));
594         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerFront));
595         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerFront));
596         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerFront));
597         iface++;
598       }
599     }
600 
601     /* Side 3 (Back) */
602     for (vz = 0; vz < faces[2]; vz++) {
603       for (vx = 0; vx < faces[0]; vx++) {
604         voffset = numFaces + vz * vertices[0] * vertices[1] + vertices[0] * (vertices[1] - 1) + vx;
605         cone[0] = voffset + vertices[0] * vertices[1];
606         cone[1] = voffset + vertices[0] * vertices[1] + 1;
607         cone[2] = voffset + 1;
608         cone[3] = voffset;
609         PetscCall(DMPlexSetCone(dm, iface, cone));
610         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBack));
611         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBack));
612         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBack));
613         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerBack));
614         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerBack));
615         iface++;
616       }
617     }
618 
619     /* Side 4 (Left) */
620     for (vz = 0; vz < faces[2]; vz++) {
621       for (vy = 0; vy < faces[1]; vy++) {
622         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0];
623         cone[0] = voffset;
624         cone[1] = voffset + vertices[0] * vertices[1];
625         cone[2] = voffset + vertices[0] * vertices[1] + vertices[0];
626         cone[3] = voffset + vertices[0];
627         PetscCall(DMPlexSetCone(dm, iface, cone));
628         PetscCall(DMSetLabelValue(dm, "marker", iface, markerLeft));
629         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerLeft));
630         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerLeft));
631         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[1] + 0, markerLeft));
632         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerLeft));
633         iface++;
634       }
635     }
636 
637     /* Side 5 (Right) */
638     for (vz = 0; vz < faces[2]; vz++) {
639       for (vy = 0; vy < faces[1]; vy++) {
640         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0] + faces[0];
641         cone[0] = voffset + vertices[0] * vertices[1];
642         cone[1] = voffset;
643         cone[2] = voffset + vertices[0];
644         cone[3] = voffset + vertices[0] * vertices[1] + vertices[0];
645         PetscCall(DMPlexSetCone(dm, iface, cone));
646         PetscCall(DMSetLabelValue(dm, "marker", iface, markerRight));
647         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerRight));
648         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerRight));
649         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerRight));
650         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerRight));
651         iface++;
652       }
653     }
654   }
655   PetscCall(DMPlexSymmetrize(dm));
656   PetscCall(DMPlexStratify(dm));
657   /* Build coordinates */
658   PetscCall(DMSetCoordinateDim(dm, 3));
659   PetscCall(DMGetCoordinateSection(dm, &coordSection));
660   PetscCall(PetscSectionSetNumFields(coordSection, 1));
661   PetscCall(PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices));
662   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 3));
663   for (v = numFaces; v < numFaces + numVertices; ++v) {
664     PetscCall(PetscSectionSetDof(coordSection, v, 3));
665     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 3));
666   }
667   PetscCall(PetscSectionSetUp(coordSection));
668   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
669   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
670   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
671   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
672   PetscCall(VecSetBlockSize(coordinates, 3));
673   PetscCall(VecSetType(coordinates, VECSTANDARD));
674   PetscCall(VecGetArray(coordinates, &coords));
675   for (vz = 0; vz <= faces[2]; ++vz) {
676     for (vy = 0; vy <= faces[1]; ++vy) {
677       for (vx = 0; vx <= faces[0]; ++vx) {
678         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 0] = lower[0] + ((upper[0] - lower[0]) / faces[0]) * vx;
679         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 1] = lower[1] + ((upper[1] - lower[1]) / faces[1]) * vy;
680         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 2] = lower[2] + ((upper[2] - lower[2]) / faces[2]) * vz;
681       }
682     }
683   }
684   PetscCall(VecRestoreArray(coordinates, &coords));
685   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
686   PetscCall(VecDestroy(&coordinates));
687   PetscFunctionReturn(0);
688 }
689 
690 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate)
691 {
692   PetscFunctionBegin;
693   PetscValidLogicalCollectiveInt(dm, dim, 2);
694   PetscCall(DMSetDimension(dm, dim - 1));
695   PetscCall(DMSetCoordinateDim(dm, dim));
696   switch (dim) {
697   case 1:
698     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(dm, lower, upper, faces));
699     break;
700   case 2:
701     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(dm, lower, upper, faces));
702     break;
703   case 3:
704     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(dm, lower, upper, faces));
705     break;
706   default:
707     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Dimension not supported: %" PetscInt_FMT, dim);
708   }
709   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
710   PetscFunctionReturn(0);
711 }
712 
713 /*@C
714   DMPlexCreateBoxSurfaceMesh - Creates a mesh on the surface of the tensor product of unit intervals (box) using tensor cells (hexahedra).
715 
716   Collective
717 
718   Input Parameters:
719 + comm        - The communicator for the DM object
720 . dim         - The spatial dimension of the box, so the resulting mesh is has dimension dim-1
721 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
722 . lower       - The lower left corner, or NULL for (0, 0, 0)
723 . upper       - The upper right corner, or NULL for (1, 1, 1)
724 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
725 
726   Output Parameter:
727 . dm  - The DM object
728 
729   Level: beginner
730 
731 .seealso: `DMSetFromOptions()`, `DMPlexCreateBoxMesh()`, `DMPlexCreateFromFile()`, `DMSetType()`, `DMCreate()`
732 @*/
733 PetscErrorCode DMPlexCreateBoxSurfaceMesh(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate, DM *dm)
734 {
735   PetscInt  fac[3] = {1, 1, 1};
736   PetscReal low[3] = {0, 0, 0};
737   PetscReal upp[3] = {1, 1, 1};
738 
739   PetscFunctionBegin;
740   PetscCall(DMCreate(comm, dm));
741   PetscCall(DMSetType(*dm, DMPLEX));
742   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(*dm, dim, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, interpolate));
743   PetscFunctionReturn(0);
744 }
745 
746 static PetscErrorCode DMPlexCreateLineMesh_Internal(DM dm, PetscInt segments, PetscReal lower, PetscReal upper, DMBoundaryType bd)
747 {
748   PetscInt     i, fStart, fEnd, numCells = 0, numVerts = 0;
749   PetscInt     numPoints[2], *coneSize, *cones, *coneOrientations;
750   PetscScalar *vertexCoords;
751   PetscReal    L, maxCell;
752   PetscBool    markerSeparate = PETSC_FALSE;
753   PetscInt     markerLeft = 1, faceMarkerLeft = 1;
754   PetscInt     markerRight = 1, faceMarkerRight = 2;
755   PetscBool    wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
756   PetscMPIInt  rank;
757 
758   PetscFunctionBegin;
759   PetscValidPointer(dm, 1);
760 
761   PetscCall(DMSetDimension(dm, 1));
762   PetscCall(DMCreateLabel(dm, "marker"));
763   PetscCall(DMCreateLabel(dm, "Face Sets"));
764 
765   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
766   if (rank == 0) numCells = segments;
767   if (rank == 0) numVerts = segments + (wrap ? 0 : 1);
768 
769   numPoints[0] = numVerts;
770   numPoints[1] = numCells;
771   PetscCall(PetscMalloc4(numCells + numVerts, &coneSize, numCells * 2, &cones, numCells + numVerts, &coneOrientations, numVerts, &vertexCoords));
772   PetscCall(PetscArrayzero(coneOrientations, numCells + numVerts));
773   for (i = 0; i < numCells; ++i) coneSize[i] = 2;
774   for (i = 0; i < numVerts; ++i) coneSize[numCells + i] = 0;
775   for (i = 0; i < numCells; ++i) {
776     cones[2 * i]     = numCells + i % numVerts;
777     cones[2 * i + 1] = numCells + (i + 1) % numVerts;
778   }
779   for (i = 0; i < numVerts; ++i) vertexCoords[i] = lower + (upper - lower) * ((PetscReal)i / (PetscReal)numCells);
780   PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
781   PetscCall(PetscFree4(coneSize, cones, coneOrientations, vertexCoords));
782 
783   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
784   if (markerSeparate) {
785     markerLeft  = faceMarkerLeft;
786     markerRight = faceMarkerRight;
787   }
788   if (!wrap && rank == 0) {
789     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
790     PetscCall(DMSetLabelValue(dm, "marker", fStart, markerLeft));
791     PetscCall(DMSetLabelValue(dm, "marker", fEnd - 1, markerRight));
792     PetscCall(DMSetLabelValue(dm, "Face Sets", fStart, faceMarkerLeft));
793     PetscCall(DMSetLabelValue(dm, "Face Sets", fEnd - 1, faceMarkerRight));
794   }
795   if (wrap) {
796     L       = upper - lower;
797     maxCell = (PetscReal)1.1 * (L / (PetscReal)PetscMax(1, segments));
798     PetscCall(DMSetPeriodicity(dm, &maxCell, &lower, &L));
799   }
800   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
801   PetscFunctionReturn(0);
802 }
803 
804 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
805 {
806   DM      boundary, vol;
807   DMLabel bdlabel;
808 
809   PetscFunctionBegin;
810   PetscValidPointer(dm, 1);
811   for (PetscInt i = 0; i < dim; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
812   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &boundary));
813   PetscCall(DMSetType(boundary, DMPLEX));
814   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(boundary, dim, faces, lower, upper, PETSC_FALSE));
815   PetscCall(DMPlexGenerate(boundary, NULL, interpolate, &vol));
816   PetscCall(DMGetLabel(vol, "marker", &bdlabel));
817   if (bdlabel) PetscCall(DMPlexLabelComplete(vol, bdlabel));
818   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_FALSE, vol));
819   PetscCall(DMPlexReplace_Internal(dm, &vol));
820   PetscCall(DMDestroy(&boundary));
821   PetscFunctionReturn(0);
822 }
823 
824 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
825 {
826   DMLabel     cutLabel  = NULL;
827   PetscInt    markerTop = 1, faceMarkerTop = 1;
828   PetscInt    markerBottom = 1, faceMarkerBottom = 1;
829   PetscInt    markerFront = 1, faceMarkerFront = 1;
830   PetscInt    markerBack = 1, faceMarkerBack = 1;
831   PetscInt    markerRight = 1, faceMarkerRight = 1;
832   PetscInt    markerLeft = 1, faceMarkerLeft = 1;
833   PetscInt    dim;
834   PetscBool   markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
835   PetscMPIInt rank;
836 
837   PetscFunctionBegin;
838   PetscCall(DMGetDimension(dm, &dim));
839   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
840   PetscCall(DMCreateLabel(dm, "marker"));
841   PetscCall(DMCreateLabel(dm, "Face Sets"));
842   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
843   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST || bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST || bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
844     if (cutMarker) {
845       PetscCall(DMCreateLabel(dm, "periodic_cut"));
846       PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
847     }
848   }
849   switch (dim) {
850   case 2:
851     faceMarkerTop    = 3;
852     faceMarkerBottom = 1;
853     faceMarkerRight  = 2;
854     faceMarkerLeft   = 4;
855     break;
856   case 3:
857     faceMarkerBottom = 1;
858     faceMarkerTop    = 2;
859     faceMarkerFront  = 3;
860     faceMarkerBack   = 4;
861     faceMarkerRight  = 5;
862     faceMarkerLeft   = 6;
863     break;
864   default:
865     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
866   }
867   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
868   if (markerSeparate) {
869     markerBottom = faceMarkerBottom;
870     markerTop    = faceMarkerTop;
871     markerFront  = faceMarkerFront;
872     markerBack   = faceMarkerBack;
873     markerRight  = faceMarkerRight;
874     markerLeft   = faceMarkerLeft;
875   }
876   {
877     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
878     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
879     const PetscInt numZEdges    = rank == 0 ? edges[2] : 0;
880     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0] + 1) : 0;
881     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1] + 1) : 0;
882     const PetscInt numZVertices = rank == 0 ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2] + 1) : 0;
883     const PetscInt numCells     = numXEdges * numYEdges * numZEdges;
884     const PetscInt numXFaces    = numYEdges * numZEdges;
885     const PetscInt numYFaces    = numXEdges * numZEdges;
886     const PetscInt numZFaces    = numXEdges * numYEdges;
887     const PetscInt numTotXFaces = numXVertices * numXFaces;
888     const PetscInt numTotYFaces = numYVertices * numYFaces;
889     const PetscInt numTotZFaces = numZVertices * numZFaces;
890     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
891     const PetscInt numTotXEdges = numXEdges * numYVertices * numZVertices;
892     const PetscInt numTotYEdges = numYEdges * numXVertices * numZVertices;
893     const PetscInt numTotZEdges = numZEdges * numXVertices * numYVertices;
894     const PetscInt numVertices  = numXVertices * numYVertices * numZVertices;
895     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
896     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
897     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
898     const PetscInt firstYFace   = firstXFace + numTotXFaces;
899     const PetscInt firstZFace   = firstYFace + numTotYFaces;
900     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
901     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
902     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
903     Vec            coordinates;
904     PetscSection   coordSection;
905     PetscScalar   *coords;
906     PetscInt       coordSize;
907     PetscInt       v, vx, vy, vz;
908     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
909 
910     PetscCall(DMPlexSetChart(dm, 0, numCells + numFaces + numEdges + numVertices));
911     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
912     for (f = firstXFace; f < firstXFace + numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
913     for (e = firstXEdge; e < firstXEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
914     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
915     /* Build cells */
916     for (fz = 0; fz < numZEdges; ++fz) {
917       for (fy = 0; fy < numYEdges; ++fy) {
918         for (fx = 0; fx < numXEdges; ++fx) {
919           PetscInt cell  = (fz * numYEdges + fy) * numXEdges + fx;
920           PetscInt faceB = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
921           PetscInt faceT = firstZFace + (fy * numXEdges + fx) * numZVertices + ((fz + 1) % numZVertices);
922           PetscInt faceF = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
923           PetscInt faceK = firstYFace + (fz * numXEdges + fx) * numYVertices + ((fy + 1) % numYVertices);
924           PetscInt faceL = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
925           PetscInt faceR = firstXFace + (fz * numYEdges + fy) * numXVertices + ((fx + 1) % numXVertices);
926           /* B,  T,  F,  K,  R,  L */
927           PetscInt ornt[6] = {-2, 0, 0, -3, 0, -2}; /* ??? */
928           PetscInt cone[6];
929 
930           /* no boundary twisting in 3D */
931           cone[0] = faceB;
932           cone[1] = faceT;
933           cone[2] = faceF;
934           cone[3] = faceK;
935           cone[4] = faceR;
936           cone[5] = faceL;
937           PetscCall(DMPlexSetCone(dm, cell, cone));
938           PetscCall(DMPlexSetConeOrientation(dm, cell, ornt));
939           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
940           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
941           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
942         }
943       }
944     }
945     /* Build x faces */
946     for (fz = 0; fz < numZEdges; ++fz) {
947       for (fy = 0; fy < numYEdges; ++fy) {
948         for (fx = 0; fx < numXVertices; ++fx) {
949           PetscInt face    = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
950           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
951           PetscInt edgeR   = firstZEdge + (((fy + 1) % numYVertices) * numXVertices + fx) * numZEdges + fz;
952           PetscInt edgeB   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
953           PetscInt edgeT   = firstYEdge + (((fz + 1) % numZVertices) * numXVertices + fx) * numYEdges + fy;
954           PetscInt ornt[4] = {0, 0, -1, -1};
955           PetscInt cone[4];
956 
957           if (dim == 3) {
958             /* markers */
959             if (bdX != DM_BOUNDARY_PERIODIC) {
960               if (fx == numXVertices - 1) {
961                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight));
962                 PetscCall(DMSetLabelValue(dm, "marker", face, markerRight));
963               } else if (fx == 0) {
964                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft));
965                 PetscCall(DMSetLabelValue(dm, "marker", face, markerLeft));
966               }
967             }
968           }
969           cone[0] = edgeB;
970           cone[1] = edgeR;
971           cone[2] = edgeT;
972           cone[3] = edgeL;
973           PetscCall(DMPlexSetCone(dm, face, cone));
974           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
975         }
976       }
977     }
978     /* Build y faces */
979     for (fz = 0; fz < numZEdges; ++fz) {
980       for (fx = 0; fx < numXEdges; ++fx) {
981         for (fy = 0; fy < numYVertices; ++fy) {
982           PetscInt face    = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
983           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
984           PetscInt edgeR   = firstZEdge + (fy * numXVertices + ((fx + 1) % numXVertices)) * numZEdges + fz;
985           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
986           PetscInt edgeT   = firstXEdge + (((fz + 1) % numZVertices) * numYVertices + fy) * numXEdges + fx;
987           PetscInt ornt[4] = {0, 0, -1, -1};
988           PetscInt cone[4];
989 
990           if (dim == 3) {
991             /* markers */
992             if (bdY != DM_BOUNDARY_PERIODIC) {
993               if (fy == numYVertices - 1) {
994                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack));
995                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBack));
996               } else if (fy == 0) {
997                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront));
998                 PetscCall(DMSetLabelValue(dm, "marker", face, markerFront));
999               }
1000             }
1001           }
1002           cone[0] = edgeB;
1003           cone[1] = edgeR;
1004           cone[2] = edgeT;
1005           cone[3] = edgeL;
1006           PetscCall(DMPlexSetCone(dm, face, cone));
1007           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1008         }
1009       }
1010     }
1011     /* Build z faces */
1012     for (fy = 0; fy < numYEdges; ++fy) {
1013       for (fx = 0; fx < numXEdges; ++fx) {
1014         for (fz = 0; fz < numZVertices; fz++) {
1015           PetscInt face    = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
1016           PetscInt edgeL   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
1017           PetscInt edgeR   = firstYEdge + (fz * numXVertices + ((fx + 1) % numXVertices)) * numYEdges + fy;
1018           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1019           PetscInt edgeT   = firstXEdge + (fz * numYVertices + ((fy + 1) % numYVertices)) * numXEdges + fx;
1020           PetscInt ornt[4] = {0, 0, -1, -1};
1021           PetscInt cone[4];
1022 
1023           if (dim == 2) {
1024             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges - 1) {
1025               edgeR += numYEdges - 1 - 2 * fy;
1026               ornt[1] = -1;
1027             }
1028             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges - 1) {
1029               edgeT += numXEdges - 1 - 2 * fx;
1030               ornt[2] = 0;
1031             }
1032             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1033             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1034           } else {
1035             /* markers */
1036             if (bdZ != DM_BOUNDARY_PERIODIC) {
1037               if (fz == numZVertices - 1) {
1038                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop));
1039                 PetscCall(DMSetLabelValue(dm, "marker", face, markerTop));
1040               } else if (fz == 0) {
1041                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom));
1042                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBottom));
1043               }
1044             }
1045           }
1046           cone[0] = edgeB;
1047           cone[1] = edgeR;
1048           cone[2] = edgeT;
1049           cone[3] = edgeL;
1050           PetscCall(DMPlexSetCone(dm, face, cone));
1051           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1052         }
1053       }
1054     }
1055     /* Build Z edges*/
1056     for (vy = 0; vy < numYVertices; vy++) {
1057       for (vx = 0; vx < numXVertices; vx++) {
1058         for (ez = 0; ez < numZEdges; ez++) {
1059           const PetscInt edge    = firstZEdge + (vy * numXVertices + vx) * numZEdges + ez;
1060           const PetscInt vertexB = firstVertex + (ez * numYVertices + vy) * numXVertices + vx;
1061           const PetscInt vertexT = firstVertex + (((ez + 1) % numZVertices) * numYVertices + vy) * numXVertices + vx;
1062           PetscInt       cone[2];
1063 
1064           cone[0] = vertexB;
1065           cone[1] = vertexT;
1066           PetscCall(DMPlexSetCone(dm, edge, cone));
1067           if (dim == 3) {
1068             if (bdX != DM_BOUNDARY_PERIODIC) {
1069               if (vx == numXVertices - 1) {
1070                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1071                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1072                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1073               } else if (vx == 0) {
1074                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1075                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1076                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1077               }
1078             }
1079             if (bdY != DM_BOUNDARY_PERIODIC) {
1080               if (vy == numYVertices - 1) {
1081                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1082                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1083                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1084               } else if (vy == 0) {
1085                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1086                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1087                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1088               }
1089             }
1090           }
1091         }
1092       }
1093     }
1094     /* Build Y edges*/
1095     for (vz = 0; vz < numZVertices; vz++) {
1096       for (vx = 0; vx < numXVertices; vx++) {
1097         for (ey = 0; ey < numYEdges; ey++) {
1098           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges - 1) ? (numXVertices - vx - 1) : (vz * numYVertices + ((ey + 1) % numYVertices)) * numXVertices + vx;
1099           const PetscInt edge    = firstYEdge + (vz * numXVertices + vx) * numYEdges + ey;
1100           const PetscInt vertexF = firstVertex + (vz * numYVertices + ey) * numXVertices + vx;
1101           const PetscInt vertexK = firstVertex + nextv;
1102           PetscInt       cone[2];
1103 
1104           cone[0] = vertexF;
1105           cone[1] = vertexK;
1106           PetscCall(DMPlexSetCone(dm, edge, cone));
1107           if (dim == 2) {
1108             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1109               if (vx == numXVertices - 1) {
1110                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight));
1111                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1112                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1113                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1114               } else if (vx == 0) {
1115                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft));
1116                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1117                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1118                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1119               }
1120             } else {
1121               if (vx == 0 && cutLabel) {
1122                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1123                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1124                 if (ey == numYEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1125               }
1126             }
1127           } else {
1128             if (bdX != DM_BOUNDARY_PERIODIC) {
1129               if (vx == numXVertices - 1) {
1130                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1131                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1132                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1133               } else if (vx == 0) {
1134                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1135                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1136                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1137               }
1138             }
1139             if (bdZ != DM_BOUNDARY_PERIODIC) {
1140               if (vz == numZVertices - 1) {
1141                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1142                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1143                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1144               } else if (vz == 0) {
1145                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1146                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1147                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1148               }
1149             }
1150           }
1151         }
1152       }
1153     }
1154     /* Build X edges*/
1155     for (vz = 0; vz < numZVertices; vz++) {
1156       for (vy = 0; vy < numYVertices; vy++) {
1157         for (ex = 0; ex < numXEdges; ex++) {
1158           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges - 1) ? (numYVertices - vy - 1) * numXVertices : (vz * numYVertices + vy) * numXVertices + (ex + 1) % numXVertices;
1159           const PetscInt edge    = firstXEdge + (vz * numYVertices + vy) * numXEdges + ex;
1160           const PetscInt vertexL = firstVertex + (vz * numYVertices + vy) * numXVertices + ex;
1161           const PetscInt vertexR = firstVertex + nextv;
1162           PetscInt       cone[2];
1163 
1164           cone[0] = vertexL;
1165           cone[1] = vertexR;
1166           PetscCall(DMPlexSetCone(dm, edge, cone));
1167           if (dim == 2) {
1168             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1169               if (vy == numYVertices - 1) {
1170                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop));
1171                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1172                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1173                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1174               } else if (vy == 0) {
1175                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom));
1176                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1177                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1178                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1179               }
1180             } else {
1181               if (vy == 0 && cutLabel) {
1182                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1183                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1184                 if (ex == numXEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1185               }
1186             }
1187           } else {
1188             if (bdY != DM_BOUNDARY_PERIODIC) {
1189               if (vy == numYVertices - 1) {
1190                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1191                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1192                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1193               } else if (vy == 0) {
1194                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1195                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1196                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1197               }
1198             }
1199             if (bdZ != DM_BOUNDARY_PERIODIC) {
1200               if (vz == numZVertices - 1) {
1201                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1202                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1203                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1204               } else if (vz == 0) {
1205                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1206                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1207                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1208               }
1209             }
1210           }
1211         }
1212       }
1213     }
1214     PetscCall(DMPlexSymmetrize(dm));
1215     PetscCall(DMPlexStratify(dm));
1216     /* Build coordinates */
1217     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1218     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1219     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1220     PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVertices));
1221     for (v = firstVertex; v < firstVertex + numVertices; ++v) {
1222       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1223       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1224     }
1225     PetscCall(PetscSectionSetUp(coordSection));
1226     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1227     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1228     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1229     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1230     PetscCall(VecSetBlockSize(coordinates, dim));
1231     PetscCall(VecSetType(coordinates, VECSTANDARD));
1232     PetscCall(VecGetArray(coordinates, &coords));
1233     for (vz = 0; vz < numZVertices; ++vz) {
1234       for (vy = 0; vy < numYVertices; ++vy) {
1235         for (vx = 0; vx < numXVertices; ++vx) {
1236           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * vx;
1237           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * vy;
1238           if (dim == 3) coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 2] = lower[2] + ((upper[2] - lower[2]) / numZEdges) * vz;
1239         }
1240       }
1241     }
1242     PetscCall(VecRestoreArray(coordinates, &coords));
1243     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1244     PetscCall(VecDestroy(&coordinates));
1245   }
1246   PetscFunctionReturn(0);
1247 }
1248 
1249 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1250 {
1251   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1252   PetscInt       fac[3] = {0, 0, 0}, d;
1253 
1254   PetscFunctionBegin;
1255   PetscValidPointer(dm, 1);
1256   PetscValidLogicalCollectiveInt(dm, dim, 2);
1257   PetscCall(DMSetDimension(dm, dim));
1258   for (d = 0; d < dim; ++d) {
1259     fac[d] = faces[d];
1260     bdt[d] = periodicity[d];
1261   }
1262   PetscCall(DMPlexCreateCubeMesh_Internal(dm, lower, upper, fac, bdt[0], bdt[1], bdt[2]));
1263   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST || periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST || (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
1264     PetscReal L[3]       = {-1., -1., 0.};
1265     PetscReal maxCell[3] = {-1., -1., 0.};
1266 
1267     for (d = 0; d < dim; ++d) {
1268       if (periodicity[d] != DM_BOUNDARY_NONE) {
1269         L[d]       = upper[d] - lower[d];
1270         maxCell[d] = 1.1 * (L[d] / PetscMax(1, faces[d]));
1271       }
1272     }
1273     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1274   }
1275   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
1276   PetscFunctionReturn(0);
1277 }
1278 
1279 static PetscErrorCode DMPlexCreateBoxMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1280 {
1281   PetscFunctionBegin;
1282   if (dim == 1) PetscCall(DMPlexCreateLineMesh_Internal(dm, faces[0], lower[0], upper[0], periodicity[0]));
1283   else if (simplex) PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1284   else PetscCall(DMPlexCreateBoxMesh_Tensor_Internal(dm, dim, faces, lower, upper, periodicity));
1285   if (!interpolate && dim > 1 && !simplex) {
1286     DM udm;
1287 
1288     PetscCall(DMPlexUninterpolate(dm, &udm));
1289     PetscCall(DMPlexCopyCoordinates(dm, udm));
1290     PetscCall(DMPlexReplace_Internal(dm, &udm));
1291   }
1292   PetscFunctionReturn(0);
1293 }
1294 
1295 /*@C
1296   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
1297 
1298   Collective
1299 
1300   Input Parameters:
1301 + comm        - The communicator for the DM object
1302 . dim         - The spatial dimension
1303 . simplex     - PETSC_TRUE for simplices, PETSC_FALSE for tensor cells
1304 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1305 . lower       - The lower left corner, or NULL for (0, 0, 0)
1306 . upper       - The upper right corner, or NULL for (1, 1, 1)
1307 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
1308 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1309 
1310   Output Parameter:
1311 . dm  - The DM object
1312 
1313   Note: If you want to customize this mesh using options, you just need to
1314 $  DMCreate(comm, &dm);
1315 $  DMSetType(dm, DMPLEX);
1316 $  DMSetFromOptions(dm);
1317 and use the options on the DMSetFromOptions() page.
1318 
1319   Here is the numbering returned for 2 faces in each direction for tensor cells:
1320 $ 10---17---11---18----12
1321 $  |         |         |
1322 $  |         |         |
1323 $ 20    2   22    3    24
1324 $  |         |         |
1325 $  |         |         |
1326 $  7---15----8---16----9
1327 $  |         |         |
1328 $  |         |         |
1329 $ 19    0   21    1   23
1330 $  |         |         |
1331 $  |         |         |
1332 $  4---13----5---14----6
1333 
1334 and for simplicial cells
1335 
1336 $ 14----8---15----9----16
1337 $  |\     5  |\      7 |
1338 $  | \       | \       |
1339 $ 13   2    14    3    15
1340 $  | 4   \   | 6   \   |
1341 $  |       \ |       \ |
1342 $ 11----6---12----7----13
1343 $  |\        |\        |
1344 $  | \    1  | \     3 |
1345 $ 10   0    11    1    12
1346 $  | 0   \   | 2   \   |
1347 $  |       \ |       \ |
1348 $  8----4----9----5----10
1349 
1350   Level: beginner
1351 
1352 .seealso: `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
1353 @*/
1354 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
1355 {
1356   PetscInt       fac[3] = {1, 1, 1};
1357   PetscReal      low[3] = {0, 0, 0};
1358   PetscReal      upp[3] = {1, 1, 1};
1359   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1360 
1361   PetscFunctionBegin;
1362   PetscCall(DMCreate(comm, dm));
1363   PetscCall(DMSetType(*dm, DMPLEX));
1364   PetscCall(DMPlexCreateBoxMesh_Internal(*dm, dim, simplex, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt, interpolate));
1365   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1366   PetscFunctionReturn(0);
1367 }
1368 
1369 static PetscErrorCode DMPlexCreateWedgeBoxMesh_Internal(DM dm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1370 {
1371   DM       bdm, vol;
1372   PetscInt i;
1373 
1374   PetscFunctionBegin;
1375   for (i = 0; i < 3; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity not yet supported");
1376   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &bdm));
1377   PetscCall(DMSetType(bdm, DMPLEX));
1378   PetscCall(DMSetDimension(bdm, 2));
1379   PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(bdm, 2, faces, lower, upper, periodicity, PETSC_TRUE));
1380   PetscCall(DMPlexExtrude(bdm, faces[2], upper[2] - lower[2], PETSC_TRUE, PETSC_FALSE, NULL, NULL, &vol));
1381   PetscCall(DMDestroy(&bdm));
1382   PetscCall(DMPlexReplace_Internal(dm, &vol));
1383   if (lower[2] != 0.0) {
1384     Vec          v;
1385     PetscScalar *x;
1386     PetscInt     cDim, n;
1387 
1388     PetscCall(DMGetCoordinatesLocal(dm, &v));
1389     PetscCall(VecGetBlockSize(v, &cDim));
1390     PetscCall(VecGetLocalSize(v, &n));
1391     PetscCall(VecGetArray(v, &x));
1392     x += cDim;
1393     for (i = 0; i < n; i += cDim) x[i] += lower[2];
1394     PetscCall(VecRestoreArray(v, &x));
1395     PetscCall(DMSetCoordinatesLocal(dm, v));
1396   }
1397   PetscFunctionReturn(0);
1398 }
1399 
1400 /*@
1401   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tesselating the (x,y) plane and extruding in the third direction using wedge cells.
1402 
1403   Collective
1404 
1405   Input Parameters:
1406 + comm        - The communicator for the DM object
1407 . faces       - Number of faces per dimension, or NULL for (1, 1, 1)
1408 . lower       - The lower left corner, or NULL for (0, 0, 0)
1409 . upper       - The upper right corner, or NULL for (1, 1, 1)
1410 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
1411 . orderHeight - If PETSC_TRUE, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1412 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1413 
1414   Output Parameter:
1415 . dm  - The DM object
1416 
1417   Level: beginner
1418 
1419 .seealso: `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateWedgeCylinderMesh()`, `DMExtrude()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1420 @*/
1421 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool orderHeight, PetscBool interpolate, DM *dm)
1422 {
1423   PetscInt       fac[3] = {1, 1, 1};
1424   PetscReal      low[3] = {0, 0, 0};
1425   PetscReal      upp[3] = {1, 1, 1};
1426   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1427 
1428   PetscFunctionBegin;
1429   PetscCall(DMCreate(comm, dm));
1430   PetscCall(DMSetType(*dm, DMPLEX));
1431   PetscCall(DMPlexCreateWedgeBoxMesh_Internal(*dm, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt));
1432   if (!interpolate) {
1433     DM udm;
1434 
1435     PetscCall(DMPlexUninterpolate(*dm, &udm));
1436     PetscCall(DMPlexReplace_Internal(*dm, &udm));
1437   }
1438   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1439   PetscFunctionReturn(0);
1440 }
1441 
1442 /*@C
1443   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all DM options in the database.
1444 
1445   Logically Collective on dm
1446 
1447   Input Parameters:
1448 + dm - the DM context
1449 - prefix - the prefix to prepend to all option names
1450 
1451   Notes:
1452   A hyphen (-) must NOT be given at the beginning of the prefix name.
1453   The first character of all runtime options is AUTOMATICALLY the hyphen.
1454 
1455   Level: advanced
1456 
1457 .seealso: `SNESSetFromOptions()`
1458 @*/
1459 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
1460 {
1461   DM_Plex *mesh = (DM_Plex *)dm->data;
1462 
1463   PetscFunctionBegin;
1464   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1465   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, prefix));
1466   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)mesh->partitioner, prefix));
1467   PetscFunctionReturn(0);
1468 }
1469 
1470 /* Remap geometry to cylinder
1471    TODO: This only works for a single refinement, then it is broken
1472 
1473      Interior square: Linear interpolation is correct
1474      The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
1475      such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
1476 
1477        phi     = arctan(y/x)
1478        d_close = sqrt(1/8 + 1/4 sin^2(phi))
1479        d_far   = sqrt(1/2 + sin^2(phi))
1480 
1481      so we remap them using
1482 
1483        x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
1484        y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
1485 
1486      If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
1487 */
1488 static void snapToCylinder(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
1489 {
1490   const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1491   const PetscReal ds2 = 0.5 * dis;
1492 
1493   if ((PetscAbsScalar(u[0]) <= ds2) && (PetscAbsScalar(u[1]) <= ds2)) {
1494     f0[0] = u[0];
1495     f0[1] = u[1];
1496   } else {
1497     PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
1498 
1499     x    = PetscRealPart(u[0]);
1500     y    = PetscRealPart(u[1]);
1501     phi  = PetscAtan2Real(y, x);
1502     sinp = PetscSinReal(phi);
1503     cosp = PetscCosReal(phi);
1504     if ((PetscAbsReal(phi) > PETSC_PI / 4.0) && (PetscAbsReal(phi) < 3.0 * PETSC_PI / 4.0)) {
1505       dc = PetscAbsReal(ds2 / sinp);
1506       df = PetscAbsReal(dis / sinp);
1507       xc = ds2 * x / PetscAbsReal(y);
1508       yc = ds2 * PetscSignReal(y);
1509     } else {
1510       dc = PetscAbsReal(ds2 / cosp);
1511       df = PetscAbsReal(dis / cosp);
1512       xc = ds2 * PetscSignReal(x);
1513       yc = ds2 * y / PetscAbsReal(x);
1514     }
1515     f0[0] = xc + (u[0] - xc) * (1.0 - dc) / (df - dc);
1516     f0[1] = yc + (u[1] - yc) * (1.0 - dc) / (df - dc);
1517   }
1518   f0[2] = u[2];
1519 }
1520 
1521 static PetscErrorCode DMPlexCreateHexCylinderMesh_Internal(DM dm, DMBoundaryType periodicZ)
1522 {
1523   const PetscInt dim = 3;
1524   PetscInt       numCells, numVertices;
1525   PetscMPIInt    rank;
1526 
1527   PetscFunctionBegin;
1528   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1529   PetscCall(DMSetDimension(dm, dim));
1530   /* Create topology */
1531   {
1532     PetscInt cone[8], c;
1533 
1534     numCells    = rank == 0 ? 5 : 0;
1535     numVertices = rank == 0 ? 16 : 0;
1536     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1537       numCells *= 3;
1538       numVertices = rank == 0 ? 24 : 0;
1539     }
1540     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1541     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 8));
1542     PetscCall(DMSetUp(dm));
1543     if (rank == 0) {
1544       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1545         cone[0] = 15;
1546         cone[1] = 18;
1547         cone[2] = 17;
1548         cone[3] = 16;
1549         cone[4] = 31;
1550         cone[5] = 32;
1551         cone[6] = 33;
1552         cone[7] = 34;
1553         PetscCall(DMPlexSetCone(dm, 0, cone));
1554         cone[0] = 16;
1555         cone[1] = 17;
1556         cone[2] = 24;
1557         cone[3] = 23;
1558         cone[4] = 32;
1559         cone[5] = 36;
1560         cone[6] = 37;
1561         cone[7] = 33; /* 22 25 26 21 */
1562         PetscCall(DMPlexSetCone(dm, 1, cone));
1563         cone[0] = 18;
1564         cone[1] = 27;
1565         cone[2] = 24;
1566         cone[3] = 17;
1567         cone[4] = 34;
1568         cone[5] = 33;
1569         cone[6] = 37;
1570         cone[7] = 38;
1571         PetscCall(DMPlexSetCone(dm, 2, cone));
1572         cone[0] = 29;
1573         cone[1] = 27;
1574         cone[2] = 18;
1575         cone[3] = 15;
1576         cone[4] = 35;
1577         cone[5] = 31;
1578         cone[6] = 34;
1579         cone[7] = 38;
1580         PetscCall(DMPlexSetCone(dm, 3, cone));
1581         cone[0] = 29;
1582         cone[1] = 15;
1583         cone[2] = 16;
1584         cone[3] = 23;
1585         cone[4] = 35;
1586         cone[5] = 36;
1587         cone[6] = 32;
1588         cone[7] = 31;
1589         PetscCall(DMPlexSetCone(dm, 4, cone));
1590 
1591         cone[0] = 31;
1592         cone[1] = 34;
1593         cone[2] = 33;
1594         cone[3] = 32;
1595         cone[4] = 19;
1596         cone[5] = 22;
1597         cone[6] = 21;
1598         cone[7] = 20;
1599         PetscCall(DMPlexSetCone(dm, 5, cone));
1600         cone[0] = 32;
1601         cone[1] = 33;
1602         cone[2] = 37;
1603         cone[3] = 36;
1604         cone[4] = 22;
1605         cone[5] = 25;
1606         cone[6] = 26;
1607         cone[7] = 21;
1608         PetscCall(DMPlexSetCone(dm, 6, cone));
1609         cone[0] = 34;
1610         cone[1] = 38;
1611         cone[2] = 37;
1612         cone[3] = 33;
1613         cone[4] = 20;
1614         cone[5] = 21;
1615         cone[6] = 26;
1616         cone[7] = 28;
1617         PetscCall(DMPlexSetCone(dm, 7, cone));
1618         cone[0] = 35;
1619         cone[1] = 38;
1620         cone[2] = 34;
1621         cone[3] = 31;
1622         cone[4] = 30;
1623         cone[5] = 19;
1624         cone[6] = 20;
1625         cone[7] = 28;
1626         PetscCall(DMPlexSetCone(dm, 8, cone));
1627         cone[0] = 35;
1628         cone[1] = 31;
1629         cone[2] = 32;
1630         cone[3] = 36;
1631         cone[4] = 30;
1632         cone[5] = 25;
1633         cone[6] = 22;
1634         cone[7] = 19;
1635         PetscCall(DMPlexSetCone(dm, 9, cone));
1636 
1637         cone[0] = 19;
1638         cone[1] = 20;
1639         cone[2] = 21;
1640         cone[3] = 22;
1641         cone[4] = 15;
1642         cone[5] = 16;
1643         cone[6] = 17;
1644         cone[7] = 18;
1645         PetscCall(DMPlexSetCone(dm, 10, cone));
1646         cone[0] = 22;
1647         cone[1] = 21;
1648         cone[2] = 26;
1649         cone[3] = 25;
1650         cone[4] = 16;
1651         cone[5] = 23;
1652         cone[6] = 24;
1653         cone[7] = 17;
1654         PetscCall(DMPlexSetCone(dm, 11, cone));
1655         cone[0] = 20;
1656         cone[1] = 28;
1657         cone[2] = 26;
1658         cone[3] = 21;
1659         cone[4] = 18;
1660         cone[5] = 17;
1661         cone[6] = 24;
1662         cone[7] = 27;
1663         PetscCall(DMPlexSetCone(dm, 12, cone));
1664         cone[0] = 30;
1665         cone[1] = 28;
1666         cone[2] = 20;
1667         cone[3] = 19;
1668         cone[4] = 29;
1669         cone[5] = 15;
1670         cone[6] = 18;
1671         cone[7] = 27;
1672         PetscCall(DMPlexSetCone(dm, 13, cone));
1673         cone[0] = 30;
1674         cone[1] = 19;
1675         cone[2] = 22;
1676         cone[3] = 25;
1677         cone[4] = 29;
1678         cone[5] = 23;
1679         cone[6] = 16;
1680         cone[7] = 15;
1681         PetscCall(DMPlexSetCone(dm, 14, cone));
1682       } else {
1683         cone[0] = 5;
1684         cone[1] = 8;
1685         cone[2] = 7;
1686         cone[3] = 6;
1687         cone[4] = 9;
1688         cone[5] = 12;
1689         cone[6] = 11;
1690         cone[7] = 10;
1691         PetscCall(DMPlexSetCone(dm, 0, cone));
1692         cone[0] = 6;
1693         cone[1] = 7;
1694         cone[2] = 14;
1695         cone[3] = 13;
1696         cone[4] = 12;
1697         cone[5] = 15;
1698         cone[6] = 16;
1699         cone[7] = 11;
1700         PetscCall(DMPlexSetCone(dm, 1, cone));
1701         cone[0] = 8;
1702         cone[1] = 17;
1703         cone[2] = 14;
1704         cone[3] = 7;
1705         cone[4] = 10;
1706         cone[5] = 11;
1707         cone[6] = 16;
1708         cone[7] = 18;
1709         PetscCall(DMPlexSetCone(dm, 2, cone));
1710         cone[0] = 19;
1711         cone[1] = 17;
1712         cone[2] = 8;
1713         cone[3] = 5;
1714         cone[4] = 20;
1715         cone[5] = 9;
1716         cone[6] = 10;
1717         cone[7] = 18;
1718         PetscCall(DMPlexSetCone(dm, 3, cone));
1719         cone[0] = 19;
1720         cone[1] = 5;
1721         cone[2] = 6;
1722         cone[3] = 13;
1723         cone[4] = 20;
1724         cone[5] = 15;
1725         cone[6] = 12;
1726         cone[7] = 9;
1727         PetscCall(DMPlexSetCone(dm, 4, cone));
1728       }
1729     }
1730     PetscCall(DMPlexSymmetrize(dm));
1731     PetscCall(DMPlexStratify(dm));
1732   }
1733   /* Create cube geometry */
1734   {
1735     Vec             coordinates;
1736     PetscSection    coordSection;
1737     PetscScalar    *coords;
1738     PetscInt        coordSize, v;
1739     const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1740     const PetscReal ds2 = dis / 2.0;
1741 
1742     /* Build coordinates */
1743     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1744     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1745     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1746     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1747     for (v = numCells; v < numCells + numVertices; ++v) {
1748       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1749       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1750     }
1751     PetscCall(PetscSectionSetUp(coordSection));
1752     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1753     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1754     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1755     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1756     PetscCall(VecSetBlockSize(coordinates, dim));
1757     PetscCall(VecSetType(coordinates, VECSTANDARD));
1758     PetscCall(VecGetArray(coordinates, &coords));
1759     if (rank == 0) {
1760       coords[0 * dim + 0]  = -ds2;
1761       coords[0 * dim + 1]  = -ds2;
1762       coords[0 * dim + 2]  = 0.0;
1763       coords[1 * dim + 0]  = ds2;
1764       coords[1 * dim + 1]  = -ds2;
1765       coords[1 * dim + 2]  = 0.0;
1766       coords[2 * dim + 0]  = ds2;
1767       coords[2 * dim + 1]  = ds2;
1768       coords[2 * dim + 2]  = 0.0;
1769       coords[3 * dim + 0]  = -ds2;
1770       coords[3 * dim + 1]  = ds2;
1771       coords[3 * dim + 2]  = 0.0;
1772       coords[4 * dim + 0]  = -ds2;
1773       coords[4 * dim + 1]  = -ds2;
1774       coords[4 * dim + 2]  = 1.0;
1775       coords[5 * dim + 0]  = -ds2;
1776       coords[5 * dim + 1]  = ds2;
1777       coords[5 * dim + 2]  = 1.0;
1778       coords[6 * dim + 0]  = ds2;
1779       coords[6 * dim + 1]  = ds2;
1780       coords[6 * dim + 2]  = 1.0;
1781       coords[7 * dim + 0]  = ds2;
1782       coords[7 * dim + 1]  = -ds2;
1783       coords[7 * dim + 2]  = 1.0;
1784       coords[8 * dim + 0]  = dis;
1785       coords[8 * dim + 1]  = -dis;
1786       coords[8 * dim + 2]  = 0.0;
1787       coords[9 * dim + 0]  = dis;
1788       coords[9 * dim + 1]  = dis;
1789       coords[9 * dim + 2]  = 0.0;
1790       coords[10 * dim + 0] = dis;
1791       coords[10 * dim + 1] = -dis;
1792       coords[10 * dim + 2] = 1.0;
1793       coords[11 * dim + 0] = dis;
1794       coords[11 * dim + 1] = dis;
1795       coords[11 * dim + 2] = 1.0;
1796       coords[12 * dim + 0] = -dis;
1797       coords[12 * dim + 1] = dis;
1798       coords[12 * dim + 2] = 0.0;
1799       coords[13 * dim + 0] = -dis;
1800       coords[13 * dim + 1] = dis;
1801       coords[13 * dim + 2] = 1.0;
1802       coords[14 * dim + 0] = -dis;
1803       coords[14 * dim + 1] = -dis;
1804       coords[14 * dim + 2] = 0.0;
1805       coords[15 * dim + 0] = -dis;
1806       coords[15 * dim + 1] = -dis;
1807       coords[15 * dim + 2] = 1.0;
1808       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1809         /* 15 31 19 */ coords[16 * dim + 0] = -ds2;
1810         coords[16 * dim + 1]                = -ds2;
1811         coords[16 * dim + 2]                = 0.5;
1812         /* 16 32 22 */ coords[17 * dim + 0] = ds2;
1813         coords[17 * dim + 1]                = -ds2;
1814         coords[17 * dim + 2]                = 0.5;
1815         /* 17 33 21 */ coords[18 * dim + 0] = ds2;
1816         coords[18 * dim + 1]                = ds2;
1817         coords[18 * dim + 2]                = 0.5;
1818         /* 18 34 20 */ coords[19 * dim + 0] = -ds2;
1819         coords[19 * dim + 1]                = ds2;
1820         coords[19 * dim + 2]                = 0.5;
1821         /* 29 35 30 */ coords[20 * dim + 0] = -dis;
1822         coords[20 * dim + 1]                = -dis;
1823         coords[20 * dim + 2]                = 0.5;
1824         /* 23 36 25 */ coords[21 * dim + 0] = dis;
1825         coords[21 * dim + 1]                = -dis;
1826         coords[21 * dim + 2]                = 0.5;
1827         /* 24 37 26 */ coords[22 * dim + 0] = dis;
1828         coords[22 * dim + 1]                = dis;
1829         coords[22 * dim + 2]                = 0.5;
1830         /* 27 38 28 */ coords[23 * dim + 0] = -dis;
1831         coords[23 * dim + 1]                = dis;
1832         coords[23 * dim + 2]                = 0.5;
1833       }
1834     }
1835     PetscCall(VecRestoreArray(coordinates, &coords));
1836     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1837     PetscCall(VecDestroy(&coordinates));
1838   }
1839   /* Create periodicity */
1840   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1841     PetscReal L[3]       = {-1., -1., 0.};
1842     PetscReal maxCell[3] = {-1., -1., 0.};
1843     PetscReal lower[3]   = {0.0, 0.0, 0.0};
1844     PetscReal upper[3]   = {1.0, 1.0, 1.5};
1845     PetscInt  numZCells  = 3;
1846 
1847     L[2]       = upper[2] - lower[2];
1848     maxCell[2] = 1.1 * (L[2] / numZCells);
1849     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1850   }
1851   {
1852     DM          cdm;
1853     PetscDS     cds;
1854     PetscScalar c[2] = {1.0, 1.0};
1855 
1856     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToCylinder));
1857     PetscCall(DMGetCoordinateDM(dm, &cdm));
1858     PetscCall(DMGetDS(cdm, &cds));
1859     PetscCall(PetscDSSetConstants(cds, 2, c));
1860   }
1861   /* Wait for coordinate creation before doing in-place modification */
1862   PetscCall(DMPlexInterpolateInPlace_Internal(dm));
1863   PetscFunctionReturn(0);
1864 }
1865 
1866 /*@
1867   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
1868 
1869   Collective
1870 
1871   Input Parameters:
1872 + comm      - The communicator for the DM object
1873 - periodicZ - The boundary type for the Z direction
1874 
1875   Output Parameter:
1876 . dm  - The DM object
1877 
1878   Note:
1879   Here is the output numbering looking from the bottom of the cylinder:
1880 $       17-----14
1881 $        |     |
1882 $        |  2  |
1883 $        |     |
1884 $ 17-----8-----7-----14
1885 $  |     |     |     |
1886 $  |  3  |  0  |  1  |
1887 $  |     |     |     |
1888 $ 19-----5-----6-----13
1889 $        |     |
1890 $        |  4  |
1891 $        |     |
1892 $       19-----13
1893 $
1894 $ and up through the top
1895 $
1896 $       18-----16
1897 $        |     |
1898 $        |  2  |
1899 $        |     |
1900 $ 18----10----11-----16
1901 $  |     |     |     |
1902 $  |  3  |  0  |  1  |
1903 $  |     |     |     |
1904 $ 20-----9----12-----15
1905 $        |     |
1906 $        |  4  |
1907 $        |     |
1908 $       20-----15
1909 
1910   Level: beginner
1911 
1912 .seealso: `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1913 @*/
1914 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, DMBoundaryType periodicZ, DM *dm)
1915 {
1916   PetscFunctionBegin;
1917   PetscValidPointer(dm, 3);
1918   PetscCall(DMCreate(comm, dm));
1919   PetscCall(DMSetType(*dm, DMPLEX));
1920   PetscCall(DMPlexCreateHexCylinderMesh_Internal(*dm, periodicZ));
1921   PetscFunctionReturn(0);
1922 }
1923 
1924 static PetscErrorCode DMPlexCreateWedgeCylinderMesh_Internal(DM dm, PetscInt n, PetscBool interpolate)
1925 {
1926   const PetscInt dim = 3;
1927   PetscInt       numCells, numVertices, v;
1928   PetscMPIInt    rank;
1929 
1930   PetscFunctionBegin;
1931   PetscCheck(n >= 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %" PetscInt_FMT " cannot be negative", n);
1932   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1933   PetscCall(DMSetDimension(dm, dim));
1934   /* Must create the celltype label here so that we do not automatically try to compute the types */
1935   PetscCall(DMCreateLabel(dm, "celltype"));
1936   /* Create topology */
1937   {
1938     PetscInt cone[6], c;
1939 
1940     numCells    = rank == 0 ? n : 0;
1941     numVertices = rank == 0 ? 2 * (n + 1) : 0;
1942     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1943     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
1944     PetscCall(DMSetUp(dm));
1945     for (c = 0; c < numCells; c++) {
1946       cone[0] = c + n * 1;
1947       cone[1] = (c + 1) % n + n * 1;
1948       cone[2] = 0 + 3 * n;
1949       cone[3] = c + n * 2;
1950       cone[4] = (c + 1) % n + n * 2;
1951       cone[5] = 1 + 3 * n;
1952       PetscCall(DMPlexSetCone(dm, c, cone));
1953       PetscCall(DMPlexSetCellType(dm, c, DM_POLYTOPE_TRI_PRISM_TENSOR));
1954     }
1955     PetscCall(DMPlexSymmetrize(dm));
1956     PetscCall(DMPlexStratify(dm));
1957   }
1958   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetCellType(dm, v, DM_POLYTOPE_POINT));
1959   /* Create cylinder geometry */
1960   {
1961     Vec          coordinates;
1962     PetscSection coordSection;
1963     PetscScalar *coords;
1964     PetscInt     coordSize, c;
1965 
1966     /* Build coordinates */
1967     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1968     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1969     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1970     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1971     for (v = numCells; v < numCells + numVertices; ++v) {
1972       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1973       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1974     }
1975     PetscCall(PetscSectionSetUp(coordSection));
1976     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1977     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1978     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1979     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1980     PetscCall(VecSetBlockSize(coordinates, dim));
1981     PetscCall(VecSetType(coordinates, VECSTANDARD));
1982     PetscCall(VecGetArray(coordinates, &coords));
1983     for (c = 0; c < numCells; c++) {
1984       coords[(c + 0 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
1985       coords[(c + 0 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
1986       coords[(c + 0 * n) * dim + 2] = 1.0;
1987       coords[(c + 1 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
1988       coords[(c + 1 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
1989       coords[(c + 1 * n) * dim + 2] = 0.0;
1990     }
1991     if (rank == 0) {
1992       coords[(2 * n + 0) * dim + 0] = 0.0;
1993       coords[(2 * n + 0) * dim + 1] = 0.0;
1994       coords[(2 * n + 0) * dim + 2] = 1.0;
1995       coords[(2 * n + 1) * dim + 0] = 0.0;
1996       coords[(2 * n + 1) * dim + 1] = 0.0;
1997       coords[(2 * n + 1) * dim + 2] = 0.0;
1998     }
1999     PetscCall(VecRestoreArray(coordinates, &coords));
2000     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2001     PetscCall(VecDestroy(&coordinates));
2002   }
2003   /* Interpolate */
2004   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2005   PetscFunctionReturn(0);
2006 }
2007 
2008 /*@
2009   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
2010 
2011   Collective
2012 
2013   Input Parameters:
2014 + comm - The communicator for the DM object
2015 . n    - The number of wedges around the origin
2016 - interpolate - Create edges and faces
2017 
2018   Output Parameter:
2019 . dm  - The DM object
2020 
2021   Level: beginner
2022 
2023 .seealso: `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2024 @*/
2025 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
2026 {
2027   PetscFunctionBegin;
2028   PetscValidPointer(dm, 4);
2029   PetscCall(DMCreate(comm, dm));
2030   PetscCall(DMSetType(*dm, DMPLEX));
2031   PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(*dm, n, interpolate));
2032   PetscFunctionReturn(0);
2033 }
2034 
2035 static inline PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2036 {
2037   PetscReal prod = 0.0;
2038   PetscInt  i;
2039   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
2040   return PetscSqrtReal(prod);
2041 }
2042 static inline PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2043 {
2044   PetscReal prod = 0.0;
2045   PetscInt  i;
2046   for (i = 0; i < dim; ++i) prod += x[i] * y[i];
2047   return prod;
2048 }
2049 
2050 /* The first constant is the sphere radius */
2051 static void snapToSphere(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
2052 {
2053   PetscReal r     = PetscRealPart(constants[0]);
2054   PetscReal norm2 = 0.0, fac;
2055   PetscInt  n     = uOff[1] - uOff[0], d;
2056 
2057   for (d = 0; d < n; ++d) norm2 += PetscSqr(PetscRealPart(u[d]));
2058   fac = r / PetscSqrtReal(norm2);
2059   for (d = 0; d < n; ++d) f0[d] = u[d] * fac;
2060 }
2061 
2062 static PetscErrorCode DMPlexCreateSphereMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, PetscReal R)
2063 {
2064   const PetscInt embedDim = dim + 1;
2065   PetscSection   coordSection;
2066   Vec            coordinates;
2067   PetscScalar   *coords;
2068   PetscReal     *coordsIn;
2069   PetscInt       numCells, numEdges, numVerts = 0, firstVertex = 0, v, firstEdge, coordSize, d, c, e;
2070   PetscMPIInt    rank;
2071 
2072   PetscFunctionBegin;
2073   PetscValidLogicalCollectiveBool(dm, simplex, 3);
2074   PetscCall(DMSetDimension(dm, dim));
2075   PetscCall(DMSetCoordinateDim(dm, dim + 1));
2076   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2077   switch (dim) {
2078   case 2:
2079     if (simplex) {
2080       const PetscReal radius    = PetscSqrtReal(1 + PETSC_PHI * PETSC_PHI) / (1.0 + PETSC_PHI);
2081       const PetscReal edgeLen   = 2.0 / (1.0 + PETSC_PHI) * (R / radius);
2082       const PetscInt  degree    = 5;
2083       PetscReal       vertex[3] = {0.0, 1.0 / (1.0 + PETSC_PHI), PETSC_PHI / (1.0 + PETSC_PHI)};
2084       PetscInt        s[3]      = {1, 1, 1};
2085       PetscInt        cone[3];
2086       PetscInt       *graph, p, i, j, k;
2087 
2088       vertex[0] *= R / radius;
2089       vertex[1] *= R / radius;
2090       vertex[2] *= R / radius;
2091       numCells    = rank == 0 ? 20 : 0;
2092       numVerts    = rank == 0 ? 12 : 0;
2093       firstVertex = numCells;
2094       /* Use icosahedron, which for a R-sphere has coordinates which are all cyclic permutations of
2095 
2096            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
2097 
2098          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2099          length is then given by 2/(1+\phi) = 2 * 0.38197 = 0.76393.
2100       */
2101       /* Construct vertices */
2102       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2103       if (rank == 0) {
2104         for (p = 0, i = 0; p < embedDim; ++p) {
2105           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2106             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2107               for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertex[(d + p) % embedDim];
2108               ++i;
2109             }
2110           }
2111         }
2112       }
2113       /* Construct graph */
2114       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2115       for (i = 0; i < numVerts; ++i) {
2116         for (j = 0, k = 0; j < numVerts; ++j) {
2117           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2118             graph[i * numVerts + j] = 1;
2119             ++k;
2120           }
2121         }
2122         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid icosahedron, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2123       }
2124       /* Build Topology */
2125       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2126       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2127       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2128       /* Cells */
2129       for (i = 0, c = 0; i < numVerts; ++i) {
2130         for (j = 0; j < i; ++j) {
2131           for (k = 0; k < j; ++k) {
2132             if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i]) {
2133               cone[0] = firstVertex + i;
2134               cone[1] = firstVertex + j;
2135               cone[2] = firstVertex + k;
2136               /* Check orientation */
2137               {
2138                 const PetscInt epsilon[3][3][3] = {
2139                   {{0, 0, 0},  {0, 0, 1},  {0, -1, 0}},
2140                   {{0, 0, -1}, {0, 0, 0},  {1, 0, 0} },
2141                   {{0, 1, 0},  {-1, 0, 0}, {0, 0, 0} }
2142                 };
2143                 PetscReal normal[3];
2144                 PetscInt  e, f;
2145 
2146                 for (d = 0; d < embedDim; ++d) {
2147                   normal[d] = 0.0;
2148                   for (e = 0; e < embedDim; ++e) {
2149                     for (f = 0; f < embedDim; ++f) normal[d] += epsilon[d][e][f] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]);
2150                   }
2151                 }
2152                 if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2153                   PetscInt tmp = cone[1];
2154                   cone[1]      = cone[2];
2155                   cone[2]      = tmp;
2156                 }
2157               }
2158               PetscCall(DMPlexSetCone(dm, c++, cone));
2159             }
2160           }
2161         }
2162       }
2163       PetscCall(DMPlexSymmetrize(dm));
2164       PetscCall(DMPlexStratify(dm));
2165       PetscCall(PetscFree(graph));
2166     } else {
2167       /*
2168         12-21--13
2169          |     |
2170         25  4  24
2171          |     |
2172   12-25--9-16--8-24--13
2173    |     |     |     |
2174   23  5 17  0 15  3  22
2175    |     |     |     |
2176   10-20--6-14--7-19--11
2177          |     |
2178         20  1  19
2179          |     |
2180         10-18--11
2181          |     |
2182         23  2  22
2183          |     |
2184         12-21--13
2185        */
2186       PetscInt cone[4], ornt[4];
2187 
2188       numCells    = rank == 0 ? 6 : 0;
2189       numEdges    = rank == 0 ? 12 : 0;
2190       numVerts    = rank == 0 ? 8 : 0;
2191       firstVertex = numCells;
2192       firstEdge   = numCells + numVerts;
2193       /* Build Topology */
2194       PetscCall(DMPlexSetChart(dm, 0, numCells + numEdges + numVerts));
2195       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 4));
2196       for (e = firstEdge; e < firstEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
2197       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2198       if (rank == 0) {
2199         /* Cell 0 */
2200         cone[0] = 14;
2201         cone[1] = 15;
2202         cone[2] = 16;
2203         cone[3] = 17;
2204         PetscCall(DMPlexSetCone(dm, 0, cone));
2205         ornt[0] = 0;
2206         ornt[1] = 0;
2207         ornt[2] = 0;
2208         ornt[3] = 0;
2209         PetscCall(DMPlexSetConeOrientation(dm, 0, ornt));
2210         /* Cell 1 */
2211         cone[0] = 18;
2212         cone[1] = 19;
2213         cone[2] = 14;
2214         cone[3] = 20;
2215         PetscCall(DMPlexSetCone(dm, 1, cone));
2216         ornt[0] = 0;
2217         ornt[1] = 0;
2218         ornt[2] = -1;
2219         ornt[3] = 0;
2220         PetscCall(DMPlexSetConeOrientation(dm, 1, ornt));
2221         /* Cell 2 */
2222         cone[0] = 21;
2223         cone[1] = 22;
2224         cone[2] = 18;
2225         cone[3] = 23;
2226         PetscCall(DMPlexSetCone(dm, 2, cone));
2227         ornt[0] = 0;
2228         ornt[1] = 0;
2229         ornt[2] = -1;
2230         ornt[3] = 0;
2231         PetscCall(DMPlexSetConeOrientation(dm, 2, ornt));
2232         /* Cell 3 */
2233         cone[0] = 19;
2234         cone[1] = 22;
2235         cone[2] = 24;
2236         cone[3] = 15;
2237         PetscCall(DMPlexSetCone(dm, 3, cone));
2238         ornt[0] = -1;
2239         ornt[1] = -1;
2240         ornt[2] = 0;
2241         ornt[3] = -1;
2242         PetscCall(DMPlexSetConeOrientation(dm, 3, ornt));
2243         /* Cell 4 */
2244         cone[0] = 16;
2245         cone[1] = 24;
2246         cone[2] = 21;
2247         cone[3] = 25;
2248         PetscCall(DMPlexSetCone(dm, 4, cone));
2249         ornt[0] = -1;
2250         ornt[1] = -1;
2251         ornt[2] = -1;
2252         ornt[3] = 0;
2253         PetscCall(DMPlexSetConeOrientation(dm, 4, ornt));
2254         /* Cell 5 */
2255         cone[0] = 20;
2256         cone[1] = 17;
2257         cone[2] = 25;
2258         cone[3] = 23;
2259         PetscCall(DMPlexSetCone(dm, 5, cone));
2260         ornt[0] = -1;
2261         ornt[1] = -1;
2262         ornt[2] = -1;
2263         ornt[3] = -1;
2264         PetscCall(DMPlexSetConeOrientation(dm, 5, ornt));
2265         /* Edges */
2266         cone[0] = 6;
2267         cone[1] = 7;
2268         PetscCall(DMPlexSetCone(dm, 14, cone));
2269         cone[0] = 7;
2270         cone[1] = 8;
2271         PetscCall(DMPlexSetCone(dm, 15, cone));
2272         cone[0] = 8;
2273         cone[1] = 9;
2274         PetscCall(DMPlexSetCone(dm, 16, cone));
2275         cone[0] = 9;
2276         cone[1] = 6;
2277         PetscCall(DMPlexSetCone(dm, 17, cone));
2278         cone[0] = 10;
2279         cone[1] = 11;
2280         PetscCall(DMPlexSetCone(dm, 18, cone));
2281         cone[0] = 11;
2282         cone[1] = 7;
2283         PetscCall(DMPlexSetCone(dm, 19, cone));
2284         cone[0] = 6;
2285         cone[1] = 10;
2286         PetscCall(DMPlexSetCone(dm, 20, cone));
2287         cone[0] = 12;
2288         cone[1] = 13;
2289         PetscCall(DMPlexSetCone(dm, 21, cone));
2290         cone[0] = 13;
2291         cone[1] = 11;
2292         PetscCall(DMPlexSetCone(dm, 22, cone));
2293         cone[0] = 10;
2294         cone[1] = 12;
2295         PetscCall(DMPlexSetCone(dm, 23, cone));
2296         cone[0] = 13;
2297         cone[1] = 8;
2298         PetscCall(DMPlexSetCone(dm, 24, cone));
2299         cone[0] = 12;
2300         cone[1] = 9;
2301         PetscCall(DMPlexSetCone(dm, 25, cone));
2302       }
2303       PetscCall(DMPlexSymmetrize(dm));
2304       PetscCall(DMPlexStratify(dm));
2305       /* Build coordinates */
2306       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2307       if (rank == 0) {
2308         coordsIn[0 * embedDim + 0] = -R;
2309         coordsIn[0 * embedDim + 1] = R;
2310         coordsIn[0 * embedDim + 2] = -R;
2311         coordsIn[1 * embedDim + 0] = R;
2312         coordsIn[1 * embedDim + 1] = R;
2313         coordsIn[1 * embedDim + 2] = -R;
2314         coordsIn[2 * embedDim + 0] = R;
2315         coordsIn[2 * embedDim + 1] = -R;
2316         coordsIn[2 * embedDim + 2] = -R;
2317         coordsIn[3 * embedDim + 0] = -R;
2318         coordsIn[3 * embedDim + 1] = -R;
2319         coordsIn[3 * embedDim + 2] = -R;
2320         coordsIn[4 * embedDim + 0] = -R;
2321         coordsIn[4 * embedDim + 1] = R;
2322         coordsIn[4 * embedDim + 2] = R;
2323         coordsIn[5 * embedDim + 0] = R;
2324         coordsIn[5 * embedDim + 1] = R;
2325         coordsIn[5 * embedDim + 2] = R;
2326         coordsIn[6 * embedDim + 0] = -R;
2327         coordsIn[6 * embedDim + 1] = -R;
2328         coordsIn[6 * embedDim + 2] = R;
2329         coordsIn[7 * embedDim + 0] = R;
2330         coordsIn[7 * embedDim + 1] = -R;
2331         coordsIn[7 * embedDim + 2] = R;
2332       }
2333     }
2334     break;
2335   case 3:
2336     if (simplex) {
2337       const PetscReal edgeLen         = 1.0 / PETSC_PHI;
2338       PetscReal       vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
2339       PetscReal       vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
2340       PetscReal       vertexC[4]      = {0.5, 0.5 * PETSC_PHI, 0.5 / PETSC_PHI, 0.0};
2341       const PetscInt  degree          = 12;
2342       PetscInt        s[4]            = {1, 1, 1};
2343       PetscInt        evenPerm[12][4] = {
2344         {0, 1, 2, 3},
2345         {0, 2, 3, 1},
2346         {0, 3, 1, 2},
2347         {1, 0, 3, 2},
2348         {1, 2, 0, 3},
2349         {1, 3, 2, 0},
2350         {2, 0, 1, 3},
2351         {2, 1, 3, 0},
2352         {2, 3, 0, 1},
2353         {3, 0, 2, 1},
2354         {3, 1, 0, 2},
2355         {3, 2, 1, 0}
2356       };
2357       PetscInt  cone[4];
2358       PetscInt *graph, p, i, j, k, l;
2359 
2360       vertexA[0] *= R;
2361       vertexA[1] *= R;
2362       vertexA[2] *= R;
2363       vertexA[3] *= R;
2364       vertexB[0] *= R;
2365       vertexB[1] *= R;
2366       vertexB[2] *= R;
2367       vertexB[3] *= R;
2368       vertexC[0] *= R;
2369       vertexC[1] *= R;
2370       vertexC[2] *= R;
2371       vertexC[3] *= R;
2372       numCells    = rank == 0 ? 600 : 0;
2373       numVerts    = rank == 0 ? 120 : 0;
2374       firstVertex = numCells;
2375       /* Use the 600-cell, which for a unit sphere has coordinates which are
2376 
2377            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
2378                (\pm 1,    0,       0,      0)  all cyclic permutations   8
2379            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
2380 
2381          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2382          length is then given by 1/\phi = 0.61803.
2383 
2384          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
2385          http://mathworld.wolfram.com/600-Cell.html
2386       */
2387       /* Construct vertices */
2388       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2389       i = 0;
2390       if (rank == 0) {
2391         for (s[0] = -1; s[0] < 2; s[0] += 2) {
2392           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2393             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2394               for (s[3] = -1; s[3] < 2; s[3] += 2) {
2395                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[d] * vertexA[d];
2396                 ++i;
2397               }
2398             }
2399           }
2400         }
2401         for (p = 0; p < embedDim; ++p) {
2402           s[1] = s[2] = s[3] = 1;
2403           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2404             for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertexB[(d + p) % embedDim];
2405             ++i;
2406           }
2407         }
2408         for (p = 0; p < 12; ++p) {
2409           s[3] = 1;
2410           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2411             for (s[1] = -1; s[1] < 2; s[1] += 2) {
2412               for (s[2] = -1; s[2] < 2; s[2] += 2) {
2413                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[evenPerm[p][d]] * vertexC[evenPerm[p][d]];
2414                 ++i;
2415               }
2416             }
2417           }
2418         }
2419       }
2420       PetscCheck(i == numVerts, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertices %" PetscInt_FMT " != %" PetscInt_FMT, i, numVerts);
2421       /* Construct graph */
2422       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2423       for (i = 0; i < numVerts; ++i) {
2424         for (j = 0, k = 0; j < numVerts; ++j) {
2425           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2426             graph[i * numVerts + j] = 1;
2427             ++k;
2428           }
2429         }
2430         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2431       }
2432       /* Build Topology */
2433       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2434       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2435       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2436       /* Cells */
2437       if (rank == 0) {
2438         for (i = 0, c = 0; i < numVerts; ++i) {
2439           for (j = 0; j < i; ++j) {
2440             for (k = 0; k < j; ++k) {
2441               for (l = 0; l < k; ++l) {
2442                 if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i] && graph[l * numVerts + i] && graph[l * numVerts + j] && graph[l * numVerts + k]) {
2443                   cone[0] = firstVertex + i;
2444                   cone[1] = firstVertex + j;
2445                   cone[2] = firstVertex + k;
2446                   cone[3] = firstVertex + l;
2447                   /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
2448                   {
2449                     const PetscInt epsilon[4][4][4][4] = {
2450                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, -1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 0, 0}, {0, 1, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 1, 0}, {0, -1, 0, 0}, {0, 0, 0, 0}}},
2451 
2452                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}}, {{0, 0, -1, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}}},
2453 
2454                       {{{0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, 0, 0}, {0, -1, 0, 0}}, {{0, 0, 0, -1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 1, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}},
2455 
2456                       {{{0, 0, 0, 0}, {0, 0, -1, 0}, {0, 1, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 1, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}}, {{0, -1, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}} }
2457                     };
2458                     PetscReal normal[4];
2459                     PetscInt  e, f, g;
2460 
2461                     for (d = 0; d < embedDim; ++d) {
2462                       normal[d] = 0.0;
2463                       for (e = 0; e < embedDim; ++e) {
2464                         for (f = 0; f < embedDim; ++f) {
2465                           for (g = 0; g < embedDim; ++g) {
2466                             normal[d] += epsilon[d][e][f][g] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]) * (coordsIn[l * embedDim + f] - coordsIn[i * embedDim + f]);
2467                           }
2468                         }
2469                       }
2470                     }
2471                     if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2472                       PetscInt tmp = cone[1];
2473                       cone[1]      = cone[2];
2474                       cone[2]      = tmp;
2475                     }
2476                   }
2477                   PetscCall(DMPlexSetCone(dm, c++, cone));
2478                 }
2479               }
2480             }
2481           }
2482         }
2483       }
2484       PetscCall(DMPlexSymmetrize(dm));
2485       PetscCall(DMPlexStratify(dm));
2486       PetscCall(PetscFree(graph));
2487     }
2488     break;
2489   default:
2490     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unsupported dimension for sphere: %" PetscInt_FMT, dim);
2491   }
2492   /* Create coordinates */
2493   PetscCall(DMGetCoordinateSection(dm, &coordSection));
2494   PetscCall(PetscSectionSetNumFields(coordSection, 1));
2495   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, embedDim));
2496   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVerts));
2497   for (v = firstVertex; v < firstVertex + numVerts; ++v) {
2498     PetscCall(PetscSectionSetDof(coordSection, v, embedDim));
2499     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, embedDim));
2500   }
2501   PetscCall(PetscSectionSetUp(coordSection));
2502   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2503   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2504   PetscCall(VecSetBlockSize(coordinates, embedDim));
2505   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2506   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2507   PetscCall(VecSetType(coordinates, VECSTANDARD));
2508   PetscCall(VecGetArray(coordinates, &coords));
2509   for (v = 0; v < numVerts; ++v)
2510     for (d = 0; d < embedDim; ++d) coords[v * embedDim + d] = coordsIn[v * embedDim + d];
2511   PetscCall(VecRestoreArray(coordinates, &coords));
2512   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2513   PetscCall(VecDestroy(&coordinates));
2514   PetscCall(PetscFree(coordsIn));
2515   {
2516     DM          cdm;
2517     PetscDS     cds;
2518     PetscScalar c = R;
2519 
2520     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToSphere));
2521     PetscCall(DMGetCoordinateDM(dm, &cdm));
2522     PetscCall(DMGetDS(cdm, &cds));
2523     PetscCall(PetscDSSetConstants(cds, 1, &c));
2524   }
2525   /* Wait for coordinate creation before doing in-place modification */
2526   if (simplex) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2527   PetscFunctionReturn(0);
2528 }
2529 
2530 typedef void (*TPSEvaluateFunc)(const PetscReal[], PetscReal *, PetscReal[], PetscReal (*)[3]);
2531 
2532 /*
2533  The Schwarz P implicit surface is
2534 
2535      f(x) = cos(x0) + cos(x1) + cos(x2) = 0
2536 */
2537 static void TPSEvaluate_SchwarzP(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2538 {
2539   PetscReal c[3] = {PetscCosReal(y[0] * PETSC_PI), PetscCosReal(y[1] * PETSC_PI), PetscCosReal(y[2] * PETSC_PI)};
2540   PetscReal g[3] = {-PetscSinReal(y[0] * PETSC_PI), -PetscSinReal(y[1] * PETSC_PI), -PetscSinReal(y[2] * PETSC_PI)};
2541   f[0]           = c[0] + c[1] + c[2];
2542   for (PetscInt i = 0; i < 3; i++) {
2543     grad[i] = PETSC_PI * g[i];
2544     for (PetscInt j = 0; j < 3; j++) hess[i][j] = (i == j) ? -PetscSqr(PETSC_PI) * c[i] : 0.;
2545   }
2546 }
2547 
2548 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2549 static PetscErrorCode TPSExtrudeNormalFunc_SchwarzP(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2550 {
2551   for (PetscInt i = 0; i < 3; i++) u[i] = -PETSC_PI * PetscSinReal(x[i] * PETSC_PI);
2552   return 0;
2553 }
2554 
2555 /*
2556  The Gyroid implicit surface is
2557 
2558  f(x,y,z) = sin(pi * x) * cos (pi * (y + 1/2))  + sin(pi * (y + 1/2)) * cos(pi * (z + 1/4)) + sin(pi * (z + 1/4)) * cos(pi * x)
2559 
2560 */
2561 static void TPSEvaluate_Gyroid(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2562 {
2563   PetscReal s[3] = {PetscSinReal(PETSC_PI * y[0]), PetscSinReal(PETSC_PI * (y[1] + .5)), PetscSinReal(PETSC_PI * (y[2] + .25))};
2564   PetscReal c[3] = {PetscCosReal(PETSC_PI * y[0]), PetscCosReal(PETSC_PI * (y[1] + .5)), PetscCosReal(PETSC_PI * (y[2] + .25))};
2565   f[0]           = s[0] * c[1] + s[1] * c[2] + s[2] * c[0];
2566   grad[0]        = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2567   grad[1]        = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2568   grad[2]        = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2569   hess[0][0]     = -PetscSqr(PETSC_PI) * (s[0] * c[1] + s[2] * c[0]);
2570   hess[0][1]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2571   hess[0][2]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2572   hess[1][0]     = -PetscSqr(PETSC_PI) * (s[1] * c[2] + s[0] * c[1]);
2573   hess[1][1]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2574   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2575   hess[2][0]     = -PetscSqr(PETSC_PI) * (s[2] * c[0] + s[1] * c[2]);
2576   hess[2][1]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2577   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2578 }
2579 
2580 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2581 static PetscErrorCode TPSExtrudeNormalFunc_Gyroid(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2582 {
2583   PetscReal s[3] = {PetscSinReal(PETSC_PI * x[0]), PetscSinReal(PETSC_PI * (x[1] + .5)), PetscSinReal(PETSC_PI * (x[2] + .25))};
2584   PetscReal c[3] = {PetscCosReal(PETSC_PI * x[0]), PetscCosReal(PETSC_PI * (x[1] + .5)), PetscCosReal(PETSC_PI * (x[2] + .25))};
2585   u[0]           = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2586   u[1]           = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2587   u[2]           = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2588   return 0;
2589 }
2590 
2591 /*
2592    We wish to solve
2593 
2594          min_y || y - x ||^2  subject to f(y) = 0
2595 
2596    Let g(y) = grad(f).  The minimization problem is equivalent to asking to satisfy
2597    f(y) = 0 and (y-x) is parallel to g(y).  We do this by using Householder QR to obtain a basis for the
2598    tangent space and ask for both components in the tangent space to be zero.
2599 
2600    Take g to be a column vector and compute the "full QR" factorization Q R = g,
2601    where Q = I - 2 n n^T is a symmetric orthogonal matrix.
2602    The first column of Q is parallel to g so the remaining two columns span the null space.
2603    Let Qn = Q[:,1:] be those remaining columns.  Then Qn Qn^T is an orthogonal projector into the tangent space.
2604    Since Q is symmetric, this is equivalent to multipyling by Q and taking the last two entries.
2605    In total, we have a system of 3 equations in 3 unknowns:
2606 
2607      f(y) = 0                       1 equation
2608      Qn^T (y - x) = 0               2 equations
2609 
2610    Here, we compute the residual and Jacobian of this system.
2611 */
2612 static void TPSNearestPointResJac(TPSEvaluateFunc feval, const PetscScalar x[], const PetscScalar y[], PetscScalar res[], PetscScalar J[])
2613 {
2614   PetscReal yreal[3] = {PetscRealPart(y[0]), PetscRealPart(y[1]), PetscRealPart(y[2])};
2615   PetscReal d[3]     = {PetscRealPart(y[0] - x[0]), PetscRealPart(y[1] - x[1]), PetscRealPart(y[2] - x[2])};
2616   PetscReal f, grad[3], n[3], norm, norm_y[3], nd, nd_y[3], sign;
2617   PetscReal n_y[3][3] = {
2618     {0, 0, 0},
2619     {0, 0, 0},
2620     {0, 0, 0}
2621   };
2622 
2623   feval(yreal, &f, grad, n_y);
2624 
2625   for (PetscInt i = 0; i < 3; i++) n[i] = grad[i];
2626   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2627   for (PetscInt i = 0; i < 3; i++) norm_y[i] = 1. / norm * n[i] * n_y[i][i];
2628 
2629   // Define the Householder reflector
2630   sign = n[0] >= 0 ? 1. : -1.;
2631   n[0] += norm * sign;
2632   for (PetscInt i = 0; i < 3; i++) n_y[0][i] += norm_y[i] * sign;
2633 
2634   norm      = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2635   norm_y[0] = 1. / norm * (n[0] * n_y[0][0]);
2636   norm_y[1] = 1. / norm * (n[0] * n_y[0][1] + n[1] * n_y[1][1]);
2637   norm_y[2] = 1. / norm * (n[0] * n_y[0][2] + n[2] * n_y[2][2]);
2638 
2639   for (PetscInt i = 0; i < 3; i++) {
2640     n[i] /= norm;
2641     for (PetscInt j = 0; j < 3; j++) {
2642       // note that n[i] is n_old[i]/norm when executing the code below
2643       n_y[i][j] = n_y[i][j] / norm - n[i] / norm * norm_y[j];
2644     }
2645   }
2646 
2647   nd = n[0] * d[0] + n[1] * d[1] + n[2] * d[2];
2648   for (PetscInt i = 0; i < 3; i++) nd_y[i] = n[i] + n_y[0][i] * d[0] + n_y[1][i] * d[1] + n_y[2][i] * d[2];
2649 
2650   res[0] = f;
2651   res[1] = d[1] - 2 * n[1] * nd;
2652   res[2] = d[2] - 2 * n[2] * nd;
2653   // J[j][i] is J_{ij} (column major)
2654   for (PetscInt j = 0; j < 3; j++) {
2655     J[0 + j * 3] = grad[j];
2656     J[1 + j * 3] = (j == 1) * 1. - 2 * (n_y[1][j] * nd + n[1] * nd_y[j]);
2657     J[2 + j * 3] = (j == 2) * 1. - 2 * (n_y[2][j] * nd + n[2] * nd_y[j]);
2658   }
2659 }
2660 
2661 /*
2662    Project x to the nearest point on the implicit surface using Newton's method.
2663 */
2664 static PetscErrorCode TPSNearestPoint(TPSEvaluateFunc feval, PetscScalar x[])
2665 {
2666   PetscScalar y[3] = {x[0], x[1], x[2]}; // Initial guess
2667 
2668   PetscFunctionBegin;
2669   for (PetscInt iter = 0; iter < 10; iter++) {
2670     PetscScalar res[3], J[9];
2671     PetscReal   resnorm;
2672     TPSNearestPointResJac(feval, x, y, res, J);
2673     resnorm = PetscSqrtReal(PetscSqr(PetscRealPart(res[0])) + PetscSqr(PetscRealPart(res[1])) + PetscSqr(PetscRealPart(res[2])));
2674     if (0) { // Turn on this monitor if you need to confirm quadratic convergence
2675       PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%" PetscInt_FMT "] res [%g %g %g]\n", iter, (double)PetscRealPart(res[0]), (double)PetscRealPart(res[1]), (double)PetscRealPart(res[2])));
2676     }
2677     if (resnorm < PETSC_SMALL) break;
2678 
2679     // Take the Newton step
2680     PetscCall(PetscKernel_A_gets_inverse_A_3(J, 0., PETSC_FALSE, NULL));
2681     PetscKernel_v_gets_v_minus_A_times_w_3(y, J, res);
2682   }
2683   for (PetscInt i = 0; i < 3; i++) x[i] = y[i];
2684   PetscFunctionReturn(0);
2685 }
2686 
2687 const char *const DMPlexTPSTypes[] = {"SCHWARZ_P", "GYROID", "DMPlexTPSType", "DMPLEX_TPS_", NULL};
2688 
2689 static PetscErrorCode DMPlexCreateTPSMesh_Internal(DM dm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness)
2690 {
2691   PetscMPIInt rank;
2692   PetscInt    topoDim = 2, spaceDim = 3, numFaces = 0, numVertices = 0, numEdges = 0;
2693   PetscInt(*edges)[2] = NULL, *edgeSets = NULL;
2694   PetscInt            *cells_flat = NULL;
2695   PetscReal           *vtxCoords  = NULL;
2696   TPSEvaluateFunc      evalFunc   = NULL;
2697   PetscSimplePointFunc normalFunc = NULL;
2698   DMLabel              label;
2699 
2700   PetscFunctionBegin;
2701   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2702   PetscCheck((layers != 0) ^ (thickness == 0.), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Layers %" PetscInt_FMT " must be nonzero iff thickness %g is nonzero", layers, (double)thickness);
2703   switch (tpstype) {
2704   case DMPLEX_TPS_SCHWARZ_P:
2705     PetscCheck(!periodic || (periodic[0] == DM_BOUNDARY_NONE && periodic[1] == DM_BOUNDARY_NONE && periodic[2] == DM_BOUNDARY_NONE), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Schwarz P does not support periodic meshes");
2706     if (rank == 0) {
2707       PetscInt(*cells)[6][4][4] = NULL; // [junction, junction-face, cell, conn]
2708       PetscInt  Njunctions = 0, Ncuts = 0, Npipes[3], vcount;
2709       PetscReal L = 1;
2710 
2711       Npipes[0]   = (extent[0] + 1) * extent[1] * extent[2];
2712       Npipes[1]   = extent[0] * (extent[1] + 1) * extent[2];
2713       Npipes[2]   = extent[0] * extent[1] * (extent[2] + 1);
2714       Njunctions  = extent[0] * extent[1] * extent[2];
2715       Ncuts       = 2 * (extent[0] * extent[1] + extent[1] * extent[2] + extent[2] * extent[0]);
2716       numVertices = 4 * (Npipes[0] + Npipes[1] + Npipes[2]) + 8 * Njunctions;
2717       PetscCall(PetscMalloc1(3 * numVertices, &vtxCoords));
2718       PetscCall(PetscMalloc1(Njunctions, &cells));
2719       PetscCall(PetscMalloc1(Ncuts * 4, &edges));
2720       PetscCall(PetscMalloc1(Ncuts * 4, &edgeSets));
2721       // x-normal pipes
2722       vcount = 0;
2723       for (PetscInt i = 0; i < extent[0] + 1; i++) {
2724         for (PetscInt j = 0; j < extent[1]; j++) {
2725           for (PetscInt k = 0; k < extent[2]; k++) {
2726             for (PetscInt l = 0; l < 4; l++) {
2727               vtxCoords[vcount++] = (2 * i - 1) * L;
2728               vtxCoords[vcount++] = 2 * j * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2729               vtxCoords[vcount++] = 2 * k * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2730             }
2731           }
2732         }
2733       }
2734       // y-normal pipes
2735       for (PetscInt i = 0; i < extent[0]; i++) {
2736         for (PetscInt j = 0; j < extent[1] + 1; j++) {
2737           for (PetscInt k = 0; k < extent[2]; k++) {
2738             for (PetscInt l = 0; l < 4; l++) {
2739               vtxCoords[vcount++] = 2 * i * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2740               vtxCoords[vcount++] = (2 * j - 1) * L;
2741               vtxCoords[vcount++] = 2 * k * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2742             }
2743           }
2744         }
2745       }
2746       // z-normal pipes
2747       for (PetscInt i = 0; i < extent[0]; i++) {
2748         for (PetscInt j = 0; j < extent[1]; j++) {
2749           for (PetscInt k = 0; k < extent[2] + 1; k++) {
2750             for (PetscInt l = 0; l < 4; l++) {
2751               vtxCoords[vcount++] = 2 * i * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2752               vtxCoords[vcount++] = 2 * j * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2753               vtxCoords[vcount++] = (2 * k - 1) * L;
2754             }
2755           }
2756         }
2757       }
2758       // junctions
2759       for (PetscInt i = 0; i < extent[0]; i++) {
2760         for (PetscInt j = 0; j < extent[1]; j++) {
2761           for (PetscInt k = 0; k < extent[2]; k++) {
2762             const PetscInt J = (i * extent[1] + j) * extent[2] + k, Jvoff = (Npipes[0] + Npipes[1] + Npipes[2]) * 4 + J * 8;
2763             PetscCheck(vcount / 3 == Jvoff, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected vertex count");
2764             for (PetscInt ii = 0; ii < 2; ii++) {
2765               for (PetscInt jj = 0; jj < 2; jj++) {
2766                 for (PetscInt kk = 0; kk < 2; kk++) {
2767                   double Ls           = (1 - sqrt(2) / 4) * L;
2768                   vtxCoords[vcount++] = 2 * i * L + (2 * ii - 1) * Ls;
2769                   vtxCoords[vcount++] = 2 * j * L + (2 * jj - 1) * Ls;
2770                   vtxCoords[vcount++] = 2 * k * L + (2 * kk - 1) * Ls;
2771                 }
2772               }
2773             }
2774             const PetscInt jfaces[3][2][4] = {
2775               {{3, 1, 0, 2}, {7, 5, 4, 6}}, // x-aligned
2776               {{5, 4, 0, 1}, {7, 6, 2, 3}}, // y-aligned
2777               {{6, 2, 0, 4}, {7, 3, 1, 5}}  // z-aligned
2778             };
2779             const PetscInt pipe_lo[3] = {// vertex numbers of pipes
2780                                          ((i * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + Npipes[0] + Npipes[1]) * 4};
2781             const PetscInt pipe_hi[3] = {// vertex numbers of pipes
2782                                          (((i + 1) * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j + 1) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + 1 + Npipes[0] + Npipes[1]) * 4};
2783             for (PetscInt dir = 0; dir < 3; dir++) { // x,y,z
2784               const PetscInt ijk[3] = {i, j, k};
2785               for (PetscInt l = 0; l < 4; l++) { // rotations
2786                 cells[J][dir * 2 + 0][l][0] = pipe_lo[dir] + l;
2787                 cells[J][dir * 2 + 0][l][1] = Jvoff + jfaces[dir][0][l];
2788                 cells[J][dir * 2 + 0][l][2] = Jvoff + jfaces[dir][0][(l - 1 + 4) % 4];
2789                 cells[J][dir * 2 + 0][l][3] = pipe_lo[dir] + (l - 1 + 4) % 4;
2790                 cells[J][dir * 2 + 1][l][0] = Jvoff + jfaces[dir][1][l];
2791                 cells[J][dir * 2 + 1][l][1] = pipe_hi[dir] + l;
2792                 cells[J][dir * 2 + 1][l][2] = pipe_hi[dir] + (l - 1 + 4) % 4;
2793                 cells[J][dir * 2 + 1][l][3] = Jvoff + jfaces[dir][1][(l - 1 + 4) % 4];
2794                 if (ijk[dir] == 0) {
2795                   edges[numEdges][0] = pipe_lo[dir] + l;
2796                   edges[numEdges][1] = pipe_lo[dir] + (l + 1) % 4;
2797                   edgeSets[numEdges] = dir * 2 + 1;
2798                   numEdges++;
2799                 }
2800                 if (ijk[dir] + 1 == extent[dir]) {
2801                   edges[numEdges][0] = pipe_hi[dir] + l;
2802                   edges[numEdges][1] = pipe_hi[dir] + (l + 1) % 4;
2803                   edgeSets[numEdges] = dir * 2 + 2;
2804                   numEdges++;
2805                 }
2806               }
2807             }
2808           }
2809         }
2810       }
2811       PetscCheck(numEdges == Ncuts * 4, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Edge count %" PetscInt_FMT " incompatible with number of cuts %" PetscInt_FMT, numEdges, Ncuts);
2812       numFaces   = 24 * Njunctions;
2813       cells_flat = cells[0][0][0];
2814     }
2815     evalFunc   = TPSEvaluate_SchwarzP;
2816     normalFunc = TPSExtrudeNormalFunc_SchwarzP;
2817     break;
2818   case DMPLEX_TPS_GYROID:
2819     if (rank == 0) {
2820       // This is a coarse mesh approximation of the gyroid shifted to being the zero of the level set
2821       //
2822       //     sin(pi*x)*cos(pi*(y+1/2)) + sin(pi*(y+1/2))*cos(pi*(z+1/4)) + sin(pi*(z+1/4))*cos(x)
2823       //
2824       // on the cell [0,2]^3.
2825       //
2826       // Think about dividing that cell into four columns, and focus on the column [0,1]x[0,1]x[0,2].
2827       // If you looked at the gyroid in that column at different slices of z you would see that it kind of spins
2828       // like a boomerang:
2829       //
2830       //     z = 0          z = 1/4        z = 1/2        z = 3/4     //
2831       //     -----          -------        -------        -------     //
2832       //                                                              //
2833       //     +       +      +       +      +       +      +   \   +   //
2834       //      \                                   /            \      //
2835       //       \            `-_   _-'            /              }     //
2836       //        *-_            `-'            _-'              /      //
2837       //     +     `-+      +       +      +-'     +      +   /   +   //
2838       //                                                              //
2839       //                                                              //
2840       //     z = 1          z = 5/4        z = 3/2        z = 7/4     //
2841       //     -----          -------        -------        -------     //
2842       //                                                              //
2843       //     +-_     +      +       +      +     _-+      +   /   +   //
2844       //        `-_            _-_            _-`            /        //
2845       //           \        _-'   `-_        /              {         //
2846       //            \                       /                \        //
2847       //     +       +      +       +      +       +      +   \   +   //
2848       //
2849       //
2850       // This course mesh approximates each of these slices by two line segments,
2851       // and then connects the segments in consecutive layers with quadrilateral faces.
2852       // All of the end points of the segments are multiples of 1/4 except for the
2853       // point * in the picture for z = 0 above and the similar points in other layers.
2854       // That point is at (gamma, gamma, 0), where gamma is calculated below.
2855       //
2856       // The column  [1,2]x[1,2]x[0,2] looks the same as this column;
2857       // The columns [1,2]x[0,1]x[0,2] and [0,1]x[1,2]x[0,2] are mirror images.
2858       //
2859       // As for how this method turned into the names given to the vertices:
2860       // that was not systematic, it was just the way it worked out in my handwritten notes.
2861 
2862       PetscInt facesPerBlock = 64;
2863       PetscInt vertsPerBlock = 56;
2864       PetscInt extentPlus[3];
2865       PetscInt numBlocks, numBlocksPlus;
2866       const PetscInt A = 0, B = 1, C = 2, D = 3, E = 4, F = 5, G = 6, H = 7, II = 8, J = 9, K = 10, L = 11, M = 12, N = 13, O = 14, P = 15, Q = 16, R = 17, S = 18, T = 19, U = 20, V = 21, W = 22, X = 23, Y = 24, Z = 25, Ap = 26, Bp = 27, Cp = 28, Dp = 29, Ep = 30, Fp = 31, Gp = 32, Hp = 33, Ip = 34, Jp = 35, Kp = 36, Lp = 37, Mp = 38, Np = 39, Op = 40, Pp = 41, Qp = 42, Rp = 43, Sp = 44, Tp = 45, Up = 46, Vp = 47, Wp = 48, Xp = 49, Yp = 50, Zp = 51, Aq = 52, Bq = 53, Cq = 54, Dq = 55;
2867       const PetscInt pattern[64][4] = {
2868   /* face to vertex within the coarse discretization of a single gyroid block */
2869   /* layer 0 */
2870         {A,           C,           K,           G          },
2871         {C,           B,           II,          K          },
2872         {D,           A,           H,           L          },
2873         {B + 56 * 1,  D,           L,           J          },
2874         {E,           B + 56 * 1,  J,           N          },
2875         {A + 56 * 2,  E,           N,           H + 56 * 2 },
2876         {F,           A + 56 * 2,  G + 56 * 2,  M          },
2877         {B,           F,           M,           II         },
2878  /* layer 1 */
2879         {G,           K,           Q,           O          },
2880         {K,           II,          P,           Q          },
2881         {L,           H,           O + 56 * 1,  R          },
2882         {J,           L,           R,           P          },
2883         {N,           J,           P,           S          },
2884         {H + 56 * 2,  N,           S,           O + 56 * 3 },
2885         {M,           G + 56 * 2,  O + 56 * 2,  T          },
2886         {II,          M,           T,           P          },
2887  /* layer 2 */
2888         {O,           Q,           Y,           U          },
2889         {Q,           P,           W,           Y          },
2890         {R,           O + 56 * 1,  U + 56 * 1,  Ap         },
2891         {P,           R,           Ap,          W          },
2892         {S,           P,           X,           Bp         },
2893         {O + 56 * 3,  S,           Bp,          V + 56 * 1 },
2894         {T,           O + 56 * 2,  V,           Z          },
2895         {P,           T,           Z,           X          },
2896  /* layer 3 */
2897         {U,           Y,           Ep,          Dp         },
2898         {Y,           W,           Cp,          Ep         },
2899         {Ap,          U + 56 * 1,  Dp + 56 * 1, Gp         },
2900         {W,           Ap,          Gp,          Cp         },
2901         {Bp,          X,           Cp + 56 * 2, Fp         },
2902         {V + 56 * 1,  Bp,          Fp,          Dp + 56 * 1},
2903         {Z,           V,           Dp,          Hp         },
2904         {X,           Z,           Hp,          Cp + 56 * 2},
2905  /* layer 4 */
2906         {Dp,          Ep,          Mp,          Kp         },
2907         {Ep,          Cp,          Ip,          Mp         },
2908         {Gp,          Dp + 56 * 1, Lp,          Np         },
2909         {Cp,          Gp,          Np,          Jp         },
2910         {Fp,          Cp + 56 * 2, Jp + 56 * 2, Pp         },
2911         {Dp + 56 * 1, Fp,          Pp,          Lp         },
2912         {Hp,          Dp,          Kp,          Op         },
2913         {Cp + 56 * 2, Hp,          Op,          Ip + 56 * 2},
2914  /* layer 5 */
2915         {Kp,          Mp,          Sp,          Rp         },
2916         {Mp,          Ip,          Qp,          Sp         },
2917         {Np,          Lp,          Rp,          Tp         },
2918         {Jp,          Np,          Tp,          Qp + 56 * 1},
2919         {Pp,          Jp + 56 * 2, Qp + 56 * 3, Up         },
2920         {Lp,          Pp,          Up,          Rp         },
2921         {Op,          Kp,          Rp,          Vp         },
2922         {Ip + 56 * 2, Op,          Vp,          Qp + 56 * 2},
2923  /* layer 6 */
2924         {Rp,          Sp,          Aq,          Yp         },
2925         {Sp,          Qp,          Wp,          Aq         },
2926         {Tp,          Rp,          Yp,          Cq         },
2927         {Qp + 56 * 1, Tp,          Cq,          Wp + 56 * 1},
2928         {Up,          Qp + 56 * 3, Xp + 56 * 1, Dq         },
2929         {Rp,          Up,          Dq,          Zp         },
2930         {Vp,          Rp,          Zp,          Bq         },
2931         {Qp + 56 * 2, Vp,          Bq,          Xp         },
2932  /* layer 7 (the top is the periodic image of the bottom of layer 0) */
2933         {Yp,          Aq,          C + 56 * 4,  A + 56 * 4 },
2934         {Aq,          Wp,          B + 56 * 4,  C + 56 * 4 },
2935         {Cq,          Yp,          A + 56 * 4,  D + 56 * 4 },
2936         {Wp + 56 * 1, Cq,          D + 56 * 4,  B + 56 * 5 },
2937         {Dq,          Xp + 56 * 1, B + 56 * 5,  E + 56 * 4 },
2938         {Zp,          Dq,          E + 56 * 4,  A + 56 * 6 },
2939         {Bq,          Zp,          A + 56 * 6,  F + 56 * 4 },
2940         {Xp,          Bq,          F + 56 * 4,  B + 56 * 4 }
2941       };
2942       const PetscReal gamma                = PetscAcosReal((PetscSqrtReal(3.) - 1.) / PetscSqrtReal(2.)) / PETSC_PI;
2943       const PetscReal patternCoords[56][3] = {
2944         {1.,        0.,        0.  }, /* A  */
2945         {0.,        1.,        0.  }, /* B  */
2946         {gamma,     gamma,     0.  }, /* C  */
2947         {1 + gamma, 1 - gamma, 0.  }, /* D  */
2948         {2 - gamma, 2 - gamma, 0.  }, /* E  */
2949         {1 - gamma, 1 + gamma, 0.  }, /* F  */
2950 
2951         {.5,        0,         .25 }, /* G  */
2952         {1.5,       0.,        .25 }, /* H  */
2953         {.5,        1.,        .25 }, /* II */
2954         {1.5,       1.,        .25 }, /* J  */
2955         {.25,       .5,        .25 }, /* K  */
2956         {1.25,      .5,        .25 }, /* L  */
2957         {.75,       1.5,       .25 }, /* M  */
2958         {1.75,      1.5,       .25 }, /* N  */
2959 
2960         {0.,        0.,        .5  }, /* O  */
2961         {1.,        1.,        .5  }, /* P  */
2962         {gamma,     1 - gamma, .5  }, /* Q  */
2963         {1 + gamma, gamma,     .5  }, /* R  */
2964         {2 - gamma, 1 + gamma, .5  }, /* S  */
2965         {1 - gamma, 2 - gamma, .5  }, /* T  */
2966 
2967         {0.,        .5,        .75 }, /* U  */
2968         {0.,        1.5,       .75 }, /* V  */
2969         {1.,        .5,        .75 }, /* W  */
2970         {1.,        1.5,       .75 }, /* X  */
2971         {.5,        .75,       .75 }, /* Y  */
2972         {.5,        1.75,      .75 }, /* Z  */
2973         {1.5,       .25,       .75 }, /* Ap */
2974         {1.5,       1.25,      .75 }, /* Bp */
2975 
2976         {1.,        0.,        1.  }, /* Cp */
2977         {0.,        1.,        1.  }, /* Dp */
2978         {1 - gamma, 1 - gamma, 1.  }, /* Ep */
2979         {1 + gamma, 1 + gamma, 1.  }, /* Fp */
2980         {2 - gamma, gamma,     1.  }, /* Gp */
2981         {gamma,     2 - gamma, 1.  }, /* Hp */
2982 
2983         {.5,        0.,        1.25}, /* Ip */
2984         {1.5,       0.,        1.25}, /* Jp */
2985         {.5,        1.,        1.25}, /* Kp */
2986         {1.5,       1.,        1.25}, /* Lp */
2987         {.75,       .5,        1.25}, /* Mp */
2988         {1.75,      .5,        1.25}, /* Np */
2989         {.25,       1.5,       1.25}, /* Op */
2990         {1.25,      1.5,       1.25}, /* Pp */
2991 
2992         {0.,        0.,        1.5 }, /* Qp */
2993         {1.,        1.,        1.5 }, /* Rp */
2994         {1 - gamma, gamma,     1.5 }, /* Sp */
2995         {2 - gamma, 1 - gamma, 1.5 }, /* Tp */
2996         {1 + gamma, 2 - gamma, 1.5 }, /* Up */
2997         {gamma,     1 + gamma, 1.5 }, /* Vp */
2998 
2999         {0.,        .5,        1.75}, /* Wp */
3000         {0.,        1.5,       1.75}, /* Xp */
3001         {1.,        .5,        1.75}, /* Yp */
3002         {1.,        1.5,       1.75}, /* Zp */
3003         {.5,        .25,       1.75}, /* Aq */
3004         {.5,        1.25,      1.75}, /* Bq */
3005         {1.5,       .75,       1.75}, /* Cq */
3006         {1.5,       1.75,      1.75}, /* Dq */
3007       };
3008       PetscInt(*cells)[64][4] = NULL;
3009       PetscBool *seen;
3010       PetscInt  *vertToTrueVert;
3011       PetscInt   count;
3012 
3013       for (PetscInt i = 0; i < 3; i++) extentPlus[i] = extent[i] + 1;
3014       numBlocks = 1;
3015       for (PetscInt i = 0; i < 3; i++) numBlocks *= extent[i];
3016       numBlocksPlus = 1;
3017       for (PetscInt i = 0; i < 3; i++) numBlocksPlus *= extentPlus[i];
3018       numFaces = numBlocks * facesPerBlock;
3019       PetscCall(PetscMalloc1(numBlocks, &cells));
3020       PetscCall(PetscCalloc1(numBlocksPlus * vertsPerBlock, &seen));
3021       for (PetscInt k = 0; k < extent[2]; k++) {
3022         for (PetscInt j = 0; j < extent[1]; j++) {
3023           for (PetscInt i = 0; i < extent[0]; i++) {
3024             for (PetscInt f = 0; f < facesPerBlock; f++) {
3025               for (PetscInt v = 0; v < 4; v++) {
3026                 PetscInt vertRaw     = pattern[f][v];
3027                 PetscInt blockidx    = vertRaw / 56;
3028                 PetscInt patternvert = vertRaw % 56;
3029                 PetscInt xplus       = (blockidx & 1);
3030                 PetscInt yplus       = (blockidx & 2) >> 1;
3031                 PetscInt zplus       = (blockidx & 4) >> 2;
3032                 PetscInt zcoord      = (periodic && periodic[2] == DM_BOUNDARY_PERIODIC) ? ((k + zplus) % extent[2]) : (k + zplus);
3033                 PetscInt ycoord      = (periodic && periodic[1] == DM_BOUNDARY_PERIODIC) ? ((j + yplus) % extent[1]) : (j + yplus);
3034                 PetscInt xcoord      = (periodic && periodic[0] == DM_BOUNDARY_PERIODIC) ? ((i + xplus) % extent[0]) : (i + xplus);
3035                 PetscInt vert        = ((zcoord * extentPlus[1] + ycoord) * extentPlus[0] + xcoord) * 56 + patternvert;
3036 
3037                 cells[(k * extent[1] + j) * extent[0] + i][f][v] = vert;
3038                 seen[vert]                                       = PETSC_TRUE;
3039               }
3040             }
3041           }
3042         }
3043       }
3044       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++)
3045         if (seen[i]) numVertices++;
3046       count = 0;
3047       PetscCall(PetscMalloc1(numBlocksPlus * vertsPerBlock, &vertToTrueVert));
3048       PetscCall(PetscMalloc1(numVertices * 3, &vtxCoords));
3049       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) vertToTrueVert[i] = -1;
3050       for (PetscInt k = 0; k < extentPlus[2]; k++) {
3051         for (PetscInt j = 0; j < extentPlus[1]; j++) {
3052           for (PetscInt i = 0; i < extentPlus[0]; i++) {
3053             for (PetscInt v = 0; v < vertsPerBlock; v++) {
3054               PetscInt vIdx = ((k * extentPlus[1] + j) * extentPlus[0] + i) * vertsPerBlock + v;
3055 
3056               if (seen[vIdx]) {
3057                 PetscInt thisVert;
3058 
3059                 vertToTrueVert[vIdx] = thisVert = count++;
3060 
3061                 for (PetscInt d = 0; d < 3; d++) vtxCoords[3 * thisVert + d] = patternCoords[v][d];
3062                 vtxCoords[3 * thisVert + 0] += i * 2;
3063                 vtxCoords[3 * thisVert + 1] += j * 2;
3064                 vtxCoords[3 * thisVert + 2] += k * 2;
3065               }
3066             }
3067           }
3068         }
3069       }
3070       for (PetscInt i = 0; i < numBlocks; i++) {
3071         for (PetscInt f = 0; f < facesPerBlock; f++) {
3072           for (PetscInt v = 0; v < 4; v++) cells[i][f][v] = vertToTrueVert[cells[i][f][v]];
3073         }
3074       }
3075       PetscCall(PetscFree(vertToTrueVert));
3076       PetscCall(PetscFree(seen));
3077       cells_flat = cells[0][0];
3078       numEdges   = 0;
3079       for (PetscInt i = 0; i < numFaces; i++) {
3080         for (PetscInt e = 0; e < 4; e++) {
3081           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3082           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3083 
3084           for (PetscInt d = 0; d < 3; d++) {
3085             if (!periodic || periodic[0] != DM_BOUNDARY_PERIODIC) {
3086               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) numEdges++;
3087               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) numEdges++;
3088             }
3089           }
3090         }
3091       }
3092       PetscCall(PetscMalloc1(numEdges, &edges));
3093       PetscCall(PetscMalloc1(numEdges, &edgeSets));
3094       for (PetscInt edge = 0, i = 0; i < numFaces; i++) {
3095         for (PetscInt e = 0; e < 4; e++) {
3096           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3097           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3098 
3099           for (PetscInt d = 0; d < 3; d++) {
3100             if (!periodic || periodic[d] != DM_BOUNDARY_PERIODIC) {
3101               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) {
3102                 edges[edge][0]   = ev[0];
3103                 edges[edge][1]   = ev[1];
3104                 edgeSets[edge++] = 2 * d;
3105               }
3106               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) {
3107                 edges[edge][0]   = ev[0];
3108                 edges[edge][1]   = ev[1];
3109                 edgeSets[edge++] = 2 * d + 1;
3110               }
3111             }
3112           }
3113         }
3114       }
3115     }
3116     evalFunc   = TPSEvaluate_Gyroid;
3117     normalFunc = TPSExtrudeNormalFunc_Gyroid;
3118     break;
3119   }
3120 
3121   PetscCall(DMSetDimension(dm, topoDim));
3122   if (rank == 0) PetscCall(DMPlexBuildFromCellList(dm, numFaces, numVertices, 4, cells_flat));
3123   else PetscCall(DMPlexBuildFromCellList(dm, 0, 0, 0, NULL));
3124   PetscCall(PetscFree(cells_flat));
3125   {
3126     DM idm;
3127     PetscCall(DMPlexInterpolate(dm, &idm));
3128     PetscCall(DMPlexReplace_Internal(dm, &idm));
3129   }
3130   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, vtxCoords));
3131   else PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, NULL));
3132   PetscCall(PetscFree(vtxCoords));
3133 
3134   PetscCall(DMCreateLabel(dm, "Face Sets"));
3135   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3136   for (PetscInt e = 0; e < numEdges; e++) {
3137     PetscInt        njoin;
3138     const PetscInt *join, verts[] = {numFaces + edges[e][0], numFaces + edges[e][1]};
3139     PetscCall(DMPlexGetJoin(dm, 2, verts, &njoin, &join));
3140     PetscCheck(njoin == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected unique join of vertices %" PetscInt_FMT " and %" PetscInt_FMT, edges[e][0], edges[e][1]);
3141     PetscCall(DMLabelSetValue(label, join[0], edgeSets[e]));
3142     PetscCall(DMPlexRestoreJoin(dm, 2, verts, &njoin, &join));
3143   }
3144   PetscCall(PetscFree(edges));
3145   PetscCall(PetscFree(edgeSets));
3146   if (tps_distribute) {
3147     DM               pdm = NULL;
3148     PetscPartitioner part;
3149 
3150     PetscCall(DMPlexGetPartitioner(dm, &part));
3151     PetscCall(PetscPartitionerSetFromOptions(part));
3152     PetscCall(DMPlexDistribute(dm, 0, NULL, &pdm));
3153     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
3154     // Do not auto-distribute again
3155     PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
3156   }
3157 
3158   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3159   for (PetscInt refine = 0; refine < refinements; refine++) {
3160     PetscInt     m;
3161     DM           dmf;
3162     Vec          X;
3163     PetscScalar *x;
3164     PetscCall(DMRefine(dm, MPI_COMM_NULL, &dmf));
3165     PetscCall(DMPlexReplace_Internal(dm, &dmf));
3166 
3167     PetscCall(DMGetCoordinatesLocal(dm, &X));
3168     PetscCall(VecGetLocalSize(X, &m));
3169     PetscCall(VecGetArray(X, &x));
3170     for (PetscInt i = 0; i < m; i += 3) PetscCall(TPSNearestPoint(evalFunc, &x[i]));
3171     PetscCall(VecRestoreArray(X, &x));
3172   }
3173 
3174   // Face Sets has already been propagated to new vertices during refinement; this propagates to the initial vertices.
3175   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3176   PetscCall(DMPlexLabelComplete(dm, label));
3177 
3178   if (thickness > 0) {
3179     DM              edm, cdm, ecdm;
3180     DMPlexTransform tr;
3181     const char     *prefix;
3182     PetscOptions    options;
3183     // Code from DMPlexExtrude
3184     PetscCall(DMPlexTransformCreate(PetscObjectComm((PetscObject)dm), &tr));
3185     PetscCall(DMPlexTransformSetDM(tr, dm));
3186     PetscCall(DMPlexTransformSetType(tr, DMPLEXEXTRUDE));
3187     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
3188     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)tr, prefix));
3189     PetscCall(PetscObjectGetOptions((PetscObject)dm, &options));
3190     PetscCall(PetscObjectSetOptions((PetscObject)tr, options));
3191     PetscCall(DMPlexTransformExtrudeSetLayers(tr, layers));
3192     PetscCall(DMPlexTransformExtrudeSetThickness(tr, thickness));
3193     PetscCall(DMPlexTransformExtrudeSetTensor(tr, PETSC_FALSE));
3194     PetscCall(DMPlexTransformExtrudeSetSymmetric(tr, PETSC_TRUE));
3195     PetscCall(DMPlexTransformExtrudeSetNormalFunction(tr, normalFunc));
3196     PetscCall(DMPlexTransformSetFromOptions(tr));
3197     PetscCall(PetscObjectSetOptions((PetscObject)tr, NULL));
3198     PetscCall(DMPlexTransformSetUp(tr));
3199     PetscCall(PetscObjectViewFromOptions((PetscObject)tr, NULL, "-dm_plex_tps_transform_view"));
3200     PetscCall(DMPlexTransformApply(tr, dm, &edm));
3201     PetscCall(DMCopyDisc(dm, edm));
3202     PetscCall(DMGetCoordinateDM(dm, &cdm));
3203     PetscCall(DMGetCoordinateDM(edm, &ecdm));
3204     PetscCall(DMCopyDisc(cdm, ecdm));
3205     PetscCall(DMPlexTransformCreateDiscLabels(tr, edm));
3206     PetscCall(DMPlexTransformDestroy(&tr));
3207     if (edm) {
3208       ((DM_Plex *)edm->data)->printFEM    = ((DM_Plex *)dm->data)->printFEM;
3209       ((DM_Plex *)edm->data)->printL2     = ((DM_Plex *)dm->data)->printL2;
3210       ((DM_Plex *)edm->data)->printLocate = ((DM_Plex *)dm->data)->printLocate;
3211     }
3212     PetscCall(DMPlexReplace_Internal(dm, &edm));
3213   }
3214   PetscFunctionReturn(0);
3215 }
3216 
3217 /*@
3218   DMPlexCreateTPSMesh - Create a distributed, interpolated mesh of a triply-periodic surface
3219 
3220   Collective
3221 
3222   Input Parameters:
3223 + comm   - The communicator for the DM object
3224 . tpstype - Type of triply-periodic surface
3225 . extent - Array of length 3 containing number of periods in each direction
3226 . periodic - array of length 3 with periodicity, or NULL for non-periodic
3227 . tps_distribute - Distribute 2D manifold mesh prior to refinement and extrusion (more scalable)
3228 . refinements - Number of factor-of-2 refinements of 2D manifold mesh
3229 . layers - Number of cell layers extruded in normal direction
3230 - thickness - Thickness in normal direction
3231 
3232   Output Parameter:
3233 . dm  - The DM object
3234 
3235   Notes:
3236   This meshes the surface of the Schwarz P or Gyroid surfaces.  Schwarz P is is the simplest member of the triply-periodic minimal surfaces.
3237   https://en.wikipedia.org/wiki/Schwarz_minimal_surface#Schwarz_P_(%22Primitive%22) and can be cut with "clean" boundaries.
3238   The Gyroid (https://en.wikipedia.org/wiki/Gyroid) is another triply-periodic minimal surface with applications in additive manufacturing; it is much more difficult to "cut" since there are no planes of symmetry.
3239   Our implementation creates a very coarse mesh of the surface and refines (by 4-way splitting) as many times as requested.
3240   On each refinement, all vertices are projected to their nearest point on the surface.
3241   This projection could readily be extended to related surfaces.
3242 
3243   The face (edge) sets for the Schwarz P surface are numbered 1(-x), 2(+x), 3(-y), 4(+y), 5(-z), 6(+z).
3244   When the mesh is refined, "Face Sets" contain the new vertices (created during refinement).  Use DMPlexLabelComplete() to propagate to coarse-level vertices.
3245 
3246   References:
3247 . * - Maskery et al, Insights into the mechanical properties of several triply periodic minimal surface lattice structures made by polymer additive manufacturing, 2017. https://doi.org/10.1016/j.polymer.2017.11.049
3248 
3249   Developer Notes:
3250   The Gyroid mesh does not currently mark boundary sets.
3251 
3252   Level: beginner
3253 
3254 .seealso: `DMPlexCreateSphereMesh()`, `DMSetType()`, `DMCreate()`
3255 @*/
3256 PetscErrorCode DMPlexCreateTPSMesh(MPI_Comm comm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness, DM *dm)
3257 {
3258   PetscFunctionBegin;
3259   PetscCall(DMCreate(comm, dm));
3260   PetscCall(DMSetType(*dm, DMPLEX));
3261   PetscCall(DMPlexCreateTPSMesh_Internal(*dm, tpstype, extent, periodic, tps_distribute, refinements, layers, thickness));
3262   PetscFunctionReturn(0);
3263 }
3264 
3265 /*@
3266   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
3267 
3268   Collective
3269 
3270   Input Parameters:
3271 + comm    - The communicator for the DM object
3272 . dim     - The dimension
3273 . simplex - Use simplices, or tensor product cells
3274 - R       - The radius
3275 
3276   Output Parameter:
3277 . dm  - The DM object
3278 
3279   Level: beginner
3280 
3281 .seealso: `DMPlexCreateBallMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3282 @*/
3283 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscReal R, DM *dm)
3284 {
3285   PetscFunctionBegin;
3286   PetscValidPointer(dm, 5);
3287   PetscCall(DMCreate(comm, dm));
3288   PetscCall(DMSetType(*dm, DMPLEX));
3289   PetscCall(DMPlexCreateSphereMesh_Internal(*dm, dim, simplex, R));
3290   PetscFunctionReturn(0);
3291 }
3292 
3293 static PetscErrorCode DMPlexCreateBallMesh_Internal(DM dm, PetscInt dim, PetscReal R)
3294 {
3295   DM      sdm, vol;
3296   DMLabel bdlabel;
3297 
3298   PetscFunctionBegin;
3299   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
3300   PetscCall(DMSetType(sdm, DMPLEX));
3301   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sdm, "bd_"));
3302   PetscCall(DMPlexCreateSphereMesh_Internal(sdm, dim - 1, PETSC_TRUE, R));
3303   PetscCall(DMSetFromOptions(sdm));
3304   PetscCall(DMViewFromOptions(sdm, NULL, "-dm_view"));
3305   PetscCall(DMPlexGenerate(sdm, NULL, PETSC_TRUE, &vol));
3306   PetscCall(DMDestroy(&sdm));
3307   PetscCall(DMPlexReplace_Internal(dm, &vol));
3308   PetscCall(DMCreateLabel(dm, "marker"));
3309   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
3310   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
3311   PetscCall(DMPlexLabelComplete(dm, bdlabel));
3312   PetscFunctionReturn(0);
3313 }
3314 
3315 /*@
3316   DMPlexCreateBallMesh - Creates a simplex mesh on the d-dimensional ball, B^d.
3317 
3318   Collective
3319 
3320   Input Parameters:
3321 + comm  - The communicator for the DM object
3322 . dim   - The dimension
3323 - R     - The radius
3324 
3325   Output Parameter:
3326 . dm  - The DM object
3327 
3328   Options Database Keys:
3329 - bd_dm_refine - This will refine the surface mesh preserving the sphere geometry
3330 
3331   Level: beginner
3332 
3333 .seealso: `DMPlexCreateSphereMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3334 @*/
3335 PetscErrorCode DMPlexCreateBallMesh(MPI_Comm comm, PetscInt dim, PetscReal R, DM *dm)
3336 {
3337   PetscFunctionBegin;
3338   PetscCall(DMCreate(comm, dm));
3339   PetscCall(DMSetType(*dm, DMPLEX));
3340   PetscCall(DMPlexCreateBallMesh_Internal(*dm, dim, R));
3341   PetscFunctionReturn(0);
3342 }
3343 
3344 static PetscErrorCode DMPlexCreateReferenceCell_Internal(DM rdm, DMPolytopeType ct)
3345 {
3346   PetscFunctionBegin;
3347   switch (ct) {
3348   case DM_POLYTOPE_POINT: {
3349     PetscInt    numPoints[1]        = {1};
3350     PetscInt    coneSize[1]         = {0};
3351     PetscInt    cones[1]            = {0};
3352     PetscInt    coneOrientations[1] = {0};
3353     PetscScalar vertexCoords[1]     = {0.0};
3354 
3355     PetscCall(DMSetDimension(rdm, 0));
3356     PetscCall(DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3357   } break;
3358   case DM_POLYTOPE_SEGMENT: {
3359     PetscInt    numPoints[2]        = {2, 1};
3360     PetscInt    coneSize[3]         = {2, 0, 0};
3361     PetscInt    cones[2]            = {1, 2};
3362     PetscInt    coneOrientations[2] = {0, 0};
3363     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3364 
3365     PetscCall(DMSetDimension(rdm, 1));
3366     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3367   } break;
3368   case DM_POLYTOPE_POINT_PRISM_TENSOR: {
3369     PetscInt    numPoints[2]        = {2, 1};
3370     PetscInt    coneSize[3]         = {2, 0, 0};
3371     PetscInt    cones[2]            = {1, 2};
3372     PetscInt    coneOrientations[2] = {0, 0};
3373     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3374 
3375     PetscCall(DMSetDimension(rdm, 1));
3376     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3377   } break;
3378   case DM_POLYTOPE_TRIANGLE: {
3379     PetscInt    numPoints[2]        = {3, 1};
3380     PetscInt    coneSize[4]         = {3, 0, 0, 0};
3381     PetscInt    cones[3]            = {1, 2, 3};
3382     PetscInt    coneOrientations[3] = {0, 0, 0};
3383     PetscScalar vertexCoords[6]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0};
3384 
3385     PetscCall(DMSetDimension(rdm, 2));
3386     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3387   } break;
3388   case DM_POLYTOPE_QUADRILATERAL: {
3389     PetscInt    numPoints[2]        = {4, 1};
3390     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3391     PetscInt    cones[4]            = {1, 2, 3, 4};
3392     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3393     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0};
3394 
3395     PetscCall(DMSetDimension(rdm, 2));
3396     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3397   } break;
3398   case DM_POLYTOPE_SEG_PRISM_TENSOR: {
3399     PetscInt    numPoints[2]        = {4, 1};
3400     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3401     PetscInt    cones[4]            = {1, 2, 3, 4};
3402     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3403     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0};
3404 
3405     PetscCall(DMSetDimension(rdm, 2));
3406     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3407   } break;
3408   case DM_POLYTOPE_TETRAHEDRON: {
3409     PetscInt    numPoints[2]        = {4, 1};
3410     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3411     PetscInt    cones[4]            = {1, 2, 3, 4};
3412     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3413     PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0};
3414 
3415     PetscCall(DMSetDimension(rdm, 3));
3416     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3417   } break;
3418   case DM_POLYTOPE_HEXAHEDRON: {
3419     PetscInt    numPoints[2]        = {8, 1};
3420     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3421     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3422     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3423     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3424 
3425     PetscCall(DMSetDimension(rdm, 3));
3426     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3427   } break;
3428   case DM_POLYTOPE_TRI_PRISM: {
3429     PetscInt    numPoints[2]        = {6, 1};
3430     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3431     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3432     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3433     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3434 
3435     PetscCall(DMSetDimension(rdm, 3));
3436     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3437   } break;
3438   case DM_POLYTOPE_TRI_PRISM_TENSOR: {
3439     PetscInt    numPoints[2]        = {6, 1};
3440     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3441     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3442     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3443     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3444 
3445     PetscCall(DMSetDimension(rdm, 3));
3446     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3447   } break;
3448   case DM_POLYTOPE_QUAD_PRISM_TENSOR: {
3449     PetscInt    numPoints[2]        = {8, 1};
3450     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3451     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3452     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3453     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3454 
3455     PetscCall(DMSetDimension(rdm, 3));
3456     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3457   } break;
3458   case DM_POLYTOPE_PYRAMID: {
3459     PetscInt    numPoints[2]        = {5, 1};
3460     PetscInt    coneSize[6]         = {5, 0, 0, 0, 0, 0};
3461     PetscInt    cones[5]            = {1, 2, 3, 4, 5};
3462     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3463     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, 0.0, 0.0, 1.0};
3464 
3465     PetscCall(DMSetDimension(rdm, 3));
3466     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3467   } break;
3468   default:
3469     SETERRQ(PetscObjectComm((PetscObject)rdm), PETSC_ERR_ARG_WRONG, "Cannot create reference cell for cell type %s", DMPolytopeTypes[ct]);
3470   }
3471   {
3472     PetscInt Nv, v;
3473 
3474     /* Must create the celltype label here so that we do not automatically try to compute the types */
3475     PetscCall(DMCreateLabel(rdm, "celltype"));
3476     PetscCall(DMPlexSetCellType(rdm, 0, ct));
3477     PetscCall(DMPlexGetChart(rdm, NULL, &Nv));
3478     for (v = 1; v < Nv; ++v) PetscCall(DMPlexSetCellType(rdm, v, DM_POLYTOPE_POINT));
3479   }
3480   PetscCall(DMPlexInterpolateInPlace_Internal(rdm));
3481   PetscCall(PetscObjectSetName((PetscObject)rdm, DMPolytopeTypes[ct]));
3482   PetscFunctionReturn(0);
3483 }
3484 
3485 /*@
3486   DMPlexCreateReferenceCell - Create a DMPLEX with the appropriate FEM reference cell
3487 
3488   Collective
3489 
3490   Input Parameters:
3491 + comm - The communicator
3492 - ct   - The cell type of the reference cell
3493 
3494   Output Parameter:
3495 . refdm - The reference cell
3496 
3497   Level: intermediate
3498 
3499 .seealso: `DMPlexCreateReferenceCell()`, `DMPlexCreateBoxMesh()`
3500 @*/
3501 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, DMPolytopeType ct, DM *refdm)
3502 {
3503   PetscFunctionBegin;
3504   PetscCall(DMCreate(comm, refdm));
3505   PetscCall(DMSetType(*refdm, DMPLEX));
3506   PetscCall(DMPlexCreateReferenceCell_Internal(*refdm, ct));
3507   PetscFunctionReturn(0);
3508 }
3509 
3510 static PetscErrorCode DMPlexCreateBoundaryLabel_Private(DM dm, const char name[])
3511 {
3512   DM        plex;
3513   DMLabel   label;
3514   PetscBool hasLabel;
3515 
3516   PetscFunctionBegin;
3517   PetscCall(DMHasLabel(dm, name, &hasLabel));
3518   if (hasLabel) PetscFunctionReturn(0);
3519   PetscCall(DMCreateLabel(dm, name));
3520   PetscCall(DMGetLabel(dm, name, &label));
3521   PetscCall(DMConvert(dm, DMPLEX, &plex));
3522   PetscCall(DMPlexMarkBoundaryFaces(plex, 1, label));
3523   PetscCall(DMPlexLabelComplete(plex, label));
3524   PetscCall(DMDestroy(&plex));
3525   PetscFunctionReturn(0);
3526 }
3527 
3528 /*
3529   We use the last coordinate as the radius, the inner radius is lower[dim-1] and the outer radius is upper[dim-1]. Then we map the first coordinate around the circle.
3530 
3531     (x, y) -> (r, theta) = (x[1], (x[0] - lower[0]) * 2\pi/(upper[0] - lower[0]))
3532 */
3533 static void boxToAnnulus(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
3534 {
3535   const PetscReal low = PetscRealPart(constants[0]);
3536   const PetscReal upp = PetscRealPart(constants[1]);
3537   const PetscReal r   = PetscRealPart(u[1]);
3538   const PetscReal th  = 2. * PETSC_PI * (PetscRealPart(u[0]) - low) / (upp - low);
3539 
3540   f0[0] = r * PetscCosReal(th);
3541   f0[1] = r * PetscSinReal(th);
3542 }
3543 
3544 const char *const DMPlexShapes[] = {"box", "box_surface", "ball", "sphere", "cylinder", "schwarz_p", "gyroid", "doublet", "annulus", "unknown", "DMPlexShape", "DM_SHAPE_", NULL};
3545 
3546 static PetscErrorCode DMPlexCreateFromOptions_Internal(PetscOptionItems *PetscOptionsObject, PetscBool *useCoordSpace, DM dm)
3547 {
3548   DMPlexShape    shape   = DM_SHAPE_BOX;
3549   DMPolytopeType cell    = DM_POLYTOPE_TRIANGLE;
3550   PetscInt       dim     = 2;
3551   PetscBool      simplex = PETSC_TRUE, interpolate = PETSC_TRUE, adjCone = PETSC_FALSE, adjClosure = PETSC_TRUE, refDomain = PETSC_FALSE;
3552   PetscBool      flg, flg2, fflg, bdfflg, nameflg;
3553   MPI_Comm       comm;
3554   char           filename[PETSC_MAX_PATH_LEN]   = "<unspecified>";
3555   char           bdFilename[PETSC_MAX_PATH_LEN] = "<unspecified>";
3556   char           plexname[PETSC_MAX_PATH_LEN]   = "";
3557 
3558   PetscFunctionBegin;
3559   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3560   /* TODO Turn this into a registration interface */
3561   PetscCall(PetscOptionsString("-dm_plex_filename", "File containing a mesh", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &fflg));
3562   PetscCall(PetscOptionsString("-dm_plex_boundary_filename", "File containing a mesh boundary", "DMPlexCreateFromFile", bdFilename, bdFilename, sizeof(bdFilename), &bdfflg));
3563   PetscCall(PetscOptionsString("-dm_plex_name", "Name of the mesh in the file", "DMPlexCreateFromFile", plexname, plexname, sizeof(plexname), &nameflg));
3564   PetscCall(PetscOptionsEnum("-dm_plex_cell", "Cell shape", "", DMPolytopeTypes, (PetscEnum)cell, (PetscEnum *)&cell, NULL));
3565   PetscCall(PetscOptionsBool("-dm_plex_reference_cell_domain", "Use a reference cell domain", "", refDomain, &refDomain, NULL));
3566   PetscCall(PetscOptionsEnum("-dm_plex_shape", "Shape for built-in mesh", "", DMPlexShapes, (PetscEnum)shape, (PetscEnum *)&shape, &flg));
3567   PetscCall(PetscOptionsBoundedInt("-dm_plex_dim", "Topological dimension of the mesh", "DMGetDimension", dim, &dim, &flg, 0));
3568   PetscCheck(!(dim < 0) && !(dim > 3), comm, PETSC_ERR_ARG_OUTOFRANGE, "Dimension %" PetscInt_FMT " should be in [1, 3]", dim);
3569   PetscCall(PetscOptionsBool("-dm_plex_simplex", "Mesh cell shape", "", simplex, &simplex, &flg));
3570   PetscCall(PetscOptionsBool("-dm_plex_interpolate", "Flag to create edges and faces automatically", "", interpolate, &interpolate, &flg));
3571   PetscCall(PetscOptionsBool("-dm_plex_adj_cone", "Set adjacency direction", "DMSetBasicAdjacency", adjCone, &adjCone, &flg));
3572   PetscCall(PetscOptionsBool("-dm_plex_adj_closure", "Set adjacency size", "DMSetBasicAdjacency", adjClosure, &adjClosure, &flg2));
3573   if (flg || flg2) PetscCall(DMSetBasicAdjacency(dm, adjCone, adjClosure));
3574 
3575   switch (cell) {
3576   case DM_POLYTOPE_POINT:
3577   case DM_POLYTOPE_SEGMENT:
3578   case DM_POLYTOPE_POINT_PRISM_TENSOR:
3579   case DM_POLYTOPE_TRIANGLE:
3580   case DM_POLYTOPE_QUADRILATERAL:
3581   case DM_POLYTOPE_TETRAHEDRON:
3582   case DM_POLYTOPE_HEXAHEDRON:
3583     *useCoordSpace = PETSC_TRUE;
3584     break;
3585   default:
3586     *useCoordSpace = PETSC_FALSE;
3587     break;
3588   }
3589 
3590   if (fflg) {
3591     DM dmnew;
3592 
3593     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), filename, plexname, interpolate, &dmnew));
3594     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3595     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3596   } else if (refDomain) {
3597     PetscCall(DMPlexCreateReferenceCell_Internal(dm, cell));
3598   } else if (bdfflg) {
3599     DM bdm, dmnew;
3600 
3601     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), bdFilename, plexname, interpolate, &bdm));
3602     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)bdm, "bd_"));
3603     PetscCall(DMSetFromOptions(bdm));
3604     PetscCall(DMPlexGenerate(bdm, NULL, interpolate, &dmnew));
3605     PetscCall(DMDestroy(&bdm));
3606     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3607     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3608   } else {
3609     PetscCall(PetscObjectSetName((PetscObject)dm, DMPlexShapes[shape]));
3610     switch (shape) {
3611     case DM_SHAPE_BOX:
3612     case DM_SHAPE_ANNULUS: {
3613       PetscInt       faces[3]  = {0, 0, 0};
3614       PetscReal      lower[3]  = {0, 0, 0};
3615       PetscReal      upper[3]  = {1, 1, 1};
3616       DMBoundaryType bdt[3]    = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3617       PetscBool      isAnnular = shape == DM_SHAPE_ANNULUS ? PETSC_TRUE : PETSC_FALSE;
3618       PetscInt       i, n;
3619 
3620       n = dim;
3621       for (i = 0; i < dim; ++i) faces[i] = (dim == 1 ? 1 : 4 - dim);
3622       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3623       n = 3;
3624       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3625       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3626       n = 3;
3627       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3628       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3629       n = 3;
3630       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
3631       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3632 
3633       PetscCheck(!isAnnular || dim == 2, comm, PETSC_ERR_ARG_OUTOFRANGE, "Only two dimensional annuli have been implemented");
3634       if (isAnnular)
3635         for (i = 0; i < dim - 1; ++i) bdt[i] = DM_BOUNDARY_PERIODIC;
3636 
3637       switch (cell) {
3638       case DM_POLYTOPE_TRI_PRISM_TENSOR:
3639         PetscCall(DMPlexCreateWedgeBoxMesh_Internal(dm, faces, lower, upper, bdt));
3640         if (!interpolate) {
3641           DM udm;
3642 
3643           PetscCall(DMPlexUninterpolate(dm, &udm));
3644           PetscCall(DMPlexReplace_Internal(dm, &udm));
3645         }
3646         break;
3647       default:
3648         PetscCall(DMPlexCreateBoxMesh_Internal(dm, dim, simplex, faces, lower, upper, bdt, interpolate));
3649         break;
3650       }
3651       if (isAnnular) {
3652         DM          cdm;
3653         PetscDS     cds;
3654         PetscScalar bounds[2] = {lower[0], upper[0]};
3655 
3656         // Fix coordinates for annular region
3657         PetscCall(DMSetPeriodicity(dm, NULL, NULL, NULL));
3658         PetscCall(DMSetCellCoordinatesLocal(dm, NULL));
3659         PetscCall(DMSetCellCoordinates(dm, NULL));
3660         PetscCall(DMPlexCreateCoordinateSpace(dm, 1, NULL));
3661         PetscCall(DMGetCoordinateDM(dm, &cdm));
3662         PetscCall(DMGetDS(cdm, &cds));
3663         PetscCall(PetscDSSetConstants(cds, 2, bounds));
3664         PetscCall(DMPlexRemapGeometry(dm, 0.0, boxToAnnulus));
3665       }
3666     } break;
3667     case DM_SHAPE_BOX_SURFACE: {
3668       PetscInt  faces[3] = {0, 0, 0};
3669       PetscReal lower[3] = {0, 0, 0};
3670       PetscReal upper[3] = {1, 1, 1};
3671       PetscInt  i, n;
3672 
3673       n = dim + 1;
3674       for (i = 0; i < dim + 1; ++i) faces[i] = (dim + 1 == 1 ? 1 : 4 - (dim + 1));
3675       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3676       n = 3;
3677       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3678       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3679       n = 3;
3680       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3681       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3682       PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(dm, dim + 1, faces, lower, upper, interpolate));
3683     } break;
3684     case DM_SHAPE_SPHERE: {
3685       PetscReal R = 1.0;
3686 
3687       PetscCall(PetscOptionsReal("-dm_plex_sphere_radius", "Radius of the sphere", "", R, &R, &flg));
3688       PetscCall(DMPlexCreateSphereMesh_Internal(dm, dim, simplex, R));
3689     } break;
3690     case DM_SHAPE_BALL: {
3691       PetscReal R = 1.0;
3692 
3693       PetscCall(PetscOptionsReal("-dm_plex_ball_radius", "Radius of the ball", "", R, &R, &flg));
3694       PetscCall(DMPlexCreateBallMesh_Internal(dm, dim, R));
3695     } break;
3696     case DM_SHAPE_CYLINDER: {
3697       DMBoundaryType bdt = DM_BOUNDARY_NONE;
3698       PetscInt       Nw  = 6;
3699 
3700       PetscCall(PetscOptionsEnum("-dm_plex_cylinder_bd", "Boundary type in the z direction", "", DMBoundaryTypes, (PetscEnum)bdt, (PetscEnum *)&bdt, NULL));
3701       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_wedges", "Number of wedges around the cylinder", "", Nw, &Nw, NULL));
3702       switch (cell) {
3703       case DM_POLYTOPE_TRI_PRISM_TENSOR:
3704         PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(dm, Nw, interpolate));
3705         break;
3706       default:
3707         PetscCall(DMPlexCreateHexCylinderMesh_Internal(dm, bdt));
3708         break;
3709       }
3710     } break;
3711     case DM_SHAPE_SCHWARZ_P: // fallthrough
3712     case DM_SHAPE_GYROID: {
3713       PetscInt       extent[3] = {1, 1, 1}, refine = 0, layers = 0, three;
3714       PetscReal      thickness   = 0.;
3715       DMBoundaryType periodic[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3716       DMPlexTPSType  tps_type    = shape == DM_SHAPE_SCHWARZ_P ? DMPLEX_TPS_SCHWARZ_P : DMPLEX_TPS_GYROID;
3717       PetscBool      tps_distribute;
3718       PetscCall(PetscOptionsIntArray("-dm_plex_tps_extent", "Number of replicas for each of three dimensions", NULL, extent, (three = 3, &three), NULL));
3719       PetscCall(PetscOptionsInt("-dm_plex_tps_refine", "Number of refinements", NULL, refine, &refine, NULL));
3720       PetscCall(PetscOptionsEnumArray("-dm_plex_tps_periodic", "Periodicity in each of three dimensions", NULL, DMBoundaryTypes, (PetscEnum *)periodic, (three = 3, &three), NULL));
3721       PetscCall(PetscOptionsInt("-dm_plex_tps_layers", "Number of layers in volumetric extrusion (or zero to not extrude)", NULL, layers, &layers, NULL));
3722       PetscCall(PetscOptionsReal("-dm_plex_tps_thickness", "Thickness of volumetric extrusion", NULL, thickness, &thickness, NULL));
3723       PetscCall(DMPlexDistributeGetDefault(dm, &tps_distribute));
3724       PetscCall(PetscOptionsBool("-dm_plex_tps_distribute", "Distribute the 2D mesh prior to refinement and extrusion", NULL, tps_distribute, &tps_distribute, NULL));
3725       PetscCall(DMPlexCreateTPSMesh_Internal(dm, tps_type, extent, periodic, tps_distribute, refine, layers, thickness));
3726     } break;
3727     case DM_SHAPE_DOUBLET: {
3728       DM        dmnew;
3729       PetscReal rl = 0.0;
3730 
3731       PetscCall(PetscOptionsReal("-dm_plex_doublet_refinementlimit", "Refinement limit", NULL, rl, &rl, NULL));
3732       PetscCall(DMPlexCreateDoublet(PetscObjectComm((PetscObject)dm), dim, simplex, interpolate, rl, &dmnew));
3733       PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3734       PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3735     } break;
3736     default:
3737       SETERRQ(comm, PETSC_ERR_SUP, "Domain shape %s is unsupported", DMPlexShapes[shape]);
3738     }
3739   }
3740   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3741   if (!((PetscObject)dm)->name && nameflg) PetscCall(PetscObjectSetName((PetscObject)dm, plexname));
3742   PetscFunctionReturn(0);
3743 }
3744 
3745 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
3746 {
3747   DM_Plex  *mesh = (DM_Plex *)dm->data;
3748   PetscBool flg, flg2;
3749   char      bdLabel[PETSC_MAX_PATH_LEN];
3750 
3751   PetscFunctionBegin;
3752   /* Handle viewing */
3753   PetscCall(PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL));
3754   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fem", "Debug output level all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL, 0));
3755   PetscCall(PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL));
3756   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL, 0));
3757   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_locate", "Debug output level all point location computations", "DMLocatePoints", 0, &mesh->printLocate, NULL, 0));
3758   PetscCall(DMMonitorSetFromOptions(dm, "-dm_plex_monitor_throughput", "Monitor the simulation throughput", "DMPlexMonitorThroughput", DMPlexMonitorThroughput, NULL, &flg));
3759   if (flg) PetscCall(PetscLogDefaultBegin());
3760   /* Labeling */
3761   PetscCall(PetscOptionsString("-dm_plex_boundary_label", "Label to mark the mesh boundary", "", bdLabel, bdLabel, sizeof(bdLabel), &flg));
3762   if (flg) PetscCall(DMPlexCreateBoundaryLabel_Private(dm, bdLabel));
3763   /* Point Location */
3764   PetscCall(PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL));
3765   /* Partitioning and distribution */
3766   PetscCall(PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL));
3767   /* Generation and remeshing */
3768   PetscCall(PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL));
3769   /* Projection behavior */
3770   PetscCall(PetscOptionsBoundedInt("-dm_plex_max_projection_height", "Maximum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL, 0));
3771   PetscCall(PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL));
3772   /* Checking structure */
3773   {
3774     PetscBool all = PETSC_FALSE;
3775 
3776     PetscCall(PetscOptionsBool("-dm_plex_check_all", "Perform all basic checks", "DMPlexCheck", PETSC_FALSE, &all, NULL));
3777     if (all) {
3778       PetscCall(DMPlexCheck(dm));
3779     } else {
3780       PetscCall(PetscOptionsBool("-dm_plex_check_symmetry", "Check that the adjacency information in the mesh is symmetric", "DMPlexCheckSymmetry", PETSC_FALSE, &flg, &flg2));
3781       if (flg && flg2) PetscCall(DMPlexCheckSymmetry(dm));
3782       PetscCall(PetscOptionsBool("-dm_plex_check_skeleton", "Check that each cell has the correct number of vertices (only for homogeneous simplex or tensor meshes)", "DMPlexCheckSkeleton", PETSC_FALSE, &flg, &flg2));
3783       if (flg && flg2) PetscCall(DMPlexCheckSkeleton(dm, 0));
3784       PetscCall(PetscOptionsBool("-dm_plex_check_faces", "Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type", "DMPlexCheckFaces", PETSC_FALSE, &flg, &flg2));
3785       if (flg && flg2) PetscCall(DMPlexCheckFaces(dm, 0));
3786       PetscCall(PetscOptionsBool("-dm_plex_check_geometry", "Check that cells have positive volume", "DMPlexCheckGeometry", PETSC_FALSE, &flg, &flg2));
3787       if (flg && flg2) PetscCall(DMPlexCheckGeometry(dm));
3788       PetscCall(PetscOptionsBool("-dm_plex_check_pointsf", "Check some necessary conditions for PointSF", "DMPlexCheckPointSF", PETSC_FALSE, &flg, &flg2));
3789       if (flg && flg2) PetscCall(DMPlexCheckPointSF(dm, NULL, PETSC_FALSE));
3790       PetscCall(PetscOptionsBool("-dm_plex_check_interface_cones", "Check points on inter-partition interfaces have conforming order of cone points", "DMPlexCheckInterfaceCones", PETSC_FALSE, &flg, &flg2));
3791       if (flg && flg2) PetscCall(DMPlexCheckInterfaceCones(dm));
3792     }
3793     PetscCall(PetscOptionsBool("-dm_plex_check_cell_shape", "Check cell shape", "DMPlexCheckCellShape", PETSC_FALSE, &flg, &flg2));
3794     if (flg && flg2) PetscCall(DMPlexCheckCellShape(dm, PETSC_TRUE, PETSC_DETERMINE));
3795   }
3796   {
3797     PetscReal scale = 1.0;
3798 
3799     PetscCall(PetscOptionsReal("-dm_plex_scale", "Scale factor for mesh coordinates", "DMPlexScale", scale, &scale, &flg));
3800     if (flg) {
3801       Vec coordinates, coordinatesLocal;
3802 
3803       PetscCall(DMGetCoordinates(dm, &coordinates));
3804       PetscCall(DMGetCoordinatesLocal(dm, &coordinatesLocal));
3805       PetscCall(VecScale(coordinates, scale));
3806       PetscCall(VecScale(coordinatesLocal, scale));
3807     }
3808   }
3809   PetscCall(PetscPartitionerSetFromOptions(mesh->partitioner));
3810   PetscFunctionReturn(0);
3811 }
3812 
3813 PetscErrorCode DMSetFromOptions_Overlap_Plex(DM dm, PetscOptionItems *PetscOptionsObject, PetscInt *overlap)
3814 {
3815   PetscInt  numOvLabels = 16, numOvExLabels = 16;
3816   char     *ovLabelNames[16], *ovExLabelNames[16];
3817   PetscInt  numOvValues = 16, numOvExValues = 16, l;
3818   PetscBool flg;
3819 
3820   PetscFunctionBegin;
3821   PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", *overlap, overlap, NULL, 0));
3822   PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_labels", "List of overlap label names", "DMPlexDistribute", ovLabelNames, &numOvLabels, &flg));
3823   if (!flg) numOvLabels = 0;
3824   if (numOvLabels) {
3825     ((DM_Plex *)dm->data)->numOvLabels = numOvLabels;
3826     for (l = 0; l < numOvLabels; ++l) {
3827       PetscCall(DMGetLabel(dm, ovLabelNames[l], &((DM_Plex *)dm->data)->ovLabels[l]));
3828       PetscCheck(((DM_Plex *)dm->data)->ovLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovLabelNames[l]);
3829       PetscCall(PetscFree(ovLabelNames[l]));
3830     }
3831     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_values", "List of overlap label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovValues, &numOvValues, &flg));
3832     if (!flg) numOvValues = 0;
3833     PetscCheck(numOvLabels == numOvValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvLabels, numOvValues);
3834 
3835     PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_exclude_labels", "List of overlap exclude label names", "DMPlexDistribute", ovExLabelNames, &numOvExLabels, &flg));
3836     if (!flg) numOvExLabels = 0;
3837     ((DM_Plex *)dm->data)->numOvExLabels = numOvExLabels;
3838     for (l = 0; l < numOvExLabels; ++l) {
3839       PetscCall(DMGetLabel(dm, ovExLabelNames[l], &((DM_Plex *)dm->data)->ovExLabels[l]));
3840       PetscCheck(((DM_Plex *)dm->data)->ovExLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovExLabelNames[l]);
3841       PetscCall(PetscFree(ovExLabelNames[l]));
3842     }
3843     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_exclude_values", "List of overlap exclude label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovExValues, &numOvExValues, &flg));
3844     if (!flg) numOvExValues = 0;
3845     PetscCheck(numOvExLabels == numOvExValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of exclude labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvExLabels, numOvExValues);
3846   }
3847   PetscFunctionReturn(0);
3848 }
3849 
3850 static PetscErrorCode DMSetFromOptions_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
3851 {
3852   PetscFunctionList        ordlist;
3853   char                     oname[256];
3854   PetscReal                volume    = -1.0;
3855   PetscInt                 prerefine = 0, refine = 0, r, coarsen = 0, overlap = 0, extLayers = 0, dim;
3856   PetscBool                uniformOrig, created = PETSC_FALSE, uniform = PETSC_TRUE, distribute, interpolate = PETSC_TRUE, coordSpace = PETSC_TRUE, remap = PETSC_TRUE, ghostCells = PETSC_FALSE, isHierarchy, ignoreModel = PETSC_FALSE, flg;
3857   DMPlexReorderDefaultFlag reorder;
3858 
3859   PetscFunctionBegin;
3860   PetscOptionsHeadBegin(PetscOptionsObject, "DMPlex Options");
3861   /* Handle automatic creation */
3862   PetscCall(DMGetDimension(dm, &dim));
3863   if (dim < 0) {
3864     PetscCall(DMPlexCreateFromOptions_Internal(PetscOptionsObject, &coordSpace, dm));
3865     created = PETSC_TRUE;
3866   }
3867   PetscCall(DMGetDimension(dm, &dim));
3868   /* Handle interpolation before distribution */
3869   PetscCall(PetscOptionsBool("-dm_plex_interpolate_pre", "Flag to interpolate mesh before distribution", "", interpolate, &interpolate, &flg));
3870   if (flg) {
3871     DMPlexInterpolatedFlag interpolated;
3872 
3873     PetscCall(DMPlexIsInterpolated(dm, &interpolated));
3874     if (interpolated == DMPLEX_INTERPOLATED_FULL && !interpolate) {
3875       DM udm;
3876 
3877       PetscCall(DMPlexUninterpolate(dm, &udm));
3878       PetscCall(DMPlexReplace_Internal(dm, &udm));
3879     } else if (interpolated != DMPLEX_INTERPOLATED_FULL && interpolate) {
3880       DM idm;
3881 
3882       PetscCall(DMPlexInterpolate(dm, &idm));
3883       PetscCall(DMPlexReplace_Internal(dm, &idm));
3884     }
3885   }
3886   /* Handle DMPlex refinement before distribution */
3887   PetscCall(PetscOptionsBool("-dm_refine_ignore_model", "Flag to ignore the geometry model when refining", "DMCreate", ignoreModel, &ignoreModel, &flg));
3888   if (flg) ((DM_Plex *)dm->data)->ignoreModel = ignoreModel;
3889   PetscCall(DMPlexGetRefinementUniform(dm, &uniformOrig));
3890   PetscCall(PetscOptionsBoundedInt("-dm_refine_pre", "The number of refinements before distribution", "DMCreate", prerefine, &prerefine, NULL, 0));
3891   PetscCall(PetscOptionsBool("-dm_refine_remap_pre", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
3892   PetscCall(PetscOptionsBool("-dm_refine_uniform_pre", "Flag for uniform refinement before distribution", "DMCreate", uniform, &uniform, &flg));
3893   if (flg) PetscCall(DMPlexSetRefinementUniform(dm, uniform));
3894   PetscCall(PetscOptionsReal("-dm_refine_volume_limit_pre", "The maximum cell volume after refinement before distribution", "DMCreate", volume, &volume, &flg));
3895   if (flg) {
3896     PetscCall(DMPlexSetRefinementUniform(dm, PETSC_FALSE));
3897     PetscCall(DMPlexSetRefinementLimit(dm, volume));
3898     prerefine = PetscMax(prerefine, 1);
3899   }
3900   for (r = 0; r < prerefine; ++r) {
3901     DM             rdm;
3902     PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
3903 
3904     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3905     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
3906     PetscCall(DMPlexReplace_Internal(dm, &rdm));
3907     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3908     if (coordFunc && remap) {
3909       PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
3910       ((DM_Plex *)dm->data)->coordFunc = coordFunc;
3911     }
3912   }
3913   PetscCall(DMPlexSetRefinementUniform(dm, uniformOrig));
3914   /* Handle DMPlex extrusion before distribution */
3915   PetscCall(PetscOptionsBoundedInt("-dm_extrude", "The number of layers to extrude", "", extLayers, &extLayers, NULL, 0));
3916   if (extLayers) {
3917     DM edm;
3918 
3919     PetscCall(DMExtrude(dm, extLayers, &edm));
3920     PetscCall(DMPlexReplace_Internal(dm, &edm));
3921     ((DM_Plex *)dm->data)->coordFunc = NULL;
3922     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3923     extLayers = 0;
3924     PetscCall(DMGetDimension(dm, &dim));
3925   }
3926   /* Handle DMPlex reordering before distribution */
3927   PetscCall(DMPlexReorderGetDefault(dm, &reorder));
3928   PetscCall(MatGetOrderingList(&ordlist));
3929   PetscCall(PetscStrncpy(oname, MATORDERINGNATURAL, sizeof(oname)));
3930   PetscCall(PetscOptionsFList("-dm_plex_reorder", "Set mesh reordering type", "DMPlexGetOrdering", ordlist, MATORDERINGNATURAL, oname, sizeof(oname), &flg));
3931   if (reorder == DMPLEX_REORDER_DEFAULT_TRUE || flg) {
3932     DM pdm;
3933     IS perm;
3934 
3935     PetscCall(DMPlexGetOrdering(dm, oname, NULL, &perm));
3936     PetscCall(DMPlexPermute(dm, perm, &pdm));
3937     PetscCall(ISDestroy(&perm));
3938     PetscCall(DMPlexReplace_Internal(dm, &pdm));
3939     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3940   }
3941   /* Handle DMPlex distribution */
3942   PetscCall(DMPlexDistributeGetDefault(dm, &distribute));
3943   PetscCall(PetscOptionsBool("-dm_distribute", "Flag to redistribute a mesh among processes", "DMPlexDistribute", distribute, &distribute, NULL));
3944   PetscCall(DMSetFromOptions_Overlap_Plex(dm, PetscOptionsObject, &overlap));
3945   if (distribute) {
3946     DM               pdm = NULL;
3947     PetscPartitioner part;
3948 
3949     PetscCall(DMPlexGetPartitioner(dm, &part));
3950     PetscCall(PetscPartitionerSetFromOptions(part));
3951     PetscCall(DMPlexDistribute(dm, overlap, NULL, &pdm));
3952     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
3953   }
3954   /* Create coordinate space */
3955   if (created) {
3956     DM_Plex  *mesh   = (DM_Plex *)dm->data;
3957     PetscInt  degree = 1;
3958     PetscBool flg;
3959 
3960     PetscCall(PetscOptionsBool("-dm_coord_space", "Use an FEM space for coordinates", "", coordSpace, &coordSpace, &flg));
3961     PetscCall(PetscOptionsInt("-dm_coord_petscspace_degree", "FEM degree for coordinate space", "", degree, &degree, NULL));
3962     if (coordSpace) PetscCall(DMPlexCreateCoordinateSpace(dm, degree, mesh->coordFunc));
3963     if (flg && !coordSpace) {
3964       DM           cdm;
3965       PetscDS      cds;
3966       PetscObject  obj;
3967       PetscClassId id;
3968 
3969       PetscCall(DMGetCoordinateDM(dm, &cdm));
3970       PetscCall(DMGetDS(cdm, &cds));
3971       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
3972       PetscCall(PetscObjectGetClassId(obj, &id));
3973       if (id == PETSCFE_CLASSID) {
3974         PetscContainer dummy;
3975 
3976         PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &dummy));
3977         PetscCall(PetscObjectSetName((PetscObject)dummy, "coordinates"));
3978         PetscCall(DMSetField(cdm, 0, NULL, (PetscObject)dummy));
3979         PetscCall(PetscContainerDestroy(&dummy));
3980         PetscCall(DMClearDS(cdm));
3981       }
3982       mesh->coordFunc = NULL;
3983     }
3984     PetscCall(PetscOptionsBool("-dm_sparse_localize", "Localize only necessary cells", "", dm->sparseLocalize, &dm->sparseLocalize, &flg));
3985     PetscCall(DMLocalizeCoordinates(dm));
3986   }
3987   /* Handle DMPlex refinement */
3988   remap = PETSC_TRUE;
3989   PetscCall(PetscOptionsBoundedInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL, 0));
3990   PetscCall(PetscOptionsBool("-dm_refine_remap", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
3991   PetscCall(PetscOptionsBoundedInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy, 0));
3992   if (refine) PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3993   if (refine && isHierarchy) {
3994     DM *dms, coarseDM;
3995 
3996     PetscCall(DMGetCoarseDM(dm, &coarseDM));
3997     PetscCall(PetscObjectReference((PetscObject)coarseDM));
3998     PetscCall(PetscMalloc1(refine, &dms));
3999     PetscCall(DMRefineHierarchy(dm, refine, dms));
4000     /* Total hack since we do not pass in a pointer */
4001     PetscCall(DMPlexSwap_Static(dm, dms[refine - 1]));
4002     if (refine == 1) {
4003       PetscCall(DMSetCoarseDM(dm, dms[0]));
4004       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4005     } else {
4006       PetscCall(DMSetCoarseDM(dm, dms[refine - 2]));
4007       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4008       PetscCall(DMSetCoarseDM(dms[0], dms[refine - 1]));
4009       PetscCall(DMPlexSetRegularRefinement(dms[0], PETSC_TRUE));
4010     }
4011     PetscCall(DMSetCoarseDM(dms[refine - 1], coarseDM));
4012     PetscCall(PetscObjectDereference((PetscObject)coarseDM));
4013     /* Free DMs */
4014     for (r = 0; r < refine; ++r) {
4015       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4016       PetscCall(DMDestroy(&dms[r]));
4017     }
4018     PetscCall(PetscFree(dms));
4019   } else {
4020     for (r = 0; r < refine; ++r) {
4021       DM             rdm;
4022       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4023 
4024       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4025       PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
4026       /* Total hack since we do not pass in a pointer */
4027       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4028       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4029       if (coordFunc && remap) {
4030         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4031         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4032       }
4033     }
4034   }
4035   /* Handle DMPlex coarsening */
4036   PetscCall(PetscOptionsBoundedInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL, 0));
4037   PetscCall(PetscOptionsBoundedInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy, 0));
4038   if (coarsen && isHierarchy) {
4039     DM *dms;
4040 
4041     PetscCall(PetscMalloc1(coarsen, &dms));
4042     PetscCall(DMCoarsenHierarchy(dm, coarsen, dms));
4043     /* Free DMs */
4044     for (r = 0; r < coarsen; ++r) {
4045       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4046       PetscCall(DMDestroy(&dms[r]));
4047     }
4048     PetscCall(PetscFree(dms));
4049   } else {
4050     for (r = 0; r < coarsen; ++r) {
4051       DM             cdm;
4052       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4053 
4054       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4055       PetscCall(DMCoarsen(dm, PetscObjectComm((PetscObject)dm), &cdm));
4056       /* Total hack since we do not pass in a pointer */
4057       PetscCall(DMPlexReplace_Internal(dm, &cdm));
4058       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4059       if (coordFunc) {
4060         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4061         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4062       }
4063     }
4064   }
4065   /* Handle ghost cells */
4066   PetscCall(PetscOptionsBool("-dm_plex_create_fv_ghost_cells", "Flag to create finite volume ghost cells on the boundary", "DMCreate", ghostCells, &ghostCells, NULL));
4067   if (ghostCells) {
4068     DM   gdm;
4069     char lname[PETSC_MAX_PATH_LEN];
4070 
4071     lname[0] = '\0';
4072     PetscCall(PetscOptionsString("-dm_plex_fv_ghost_cells_label", "Label name for ghost cells boundary", "DMCreate", lname, lname, sizeof(lname), &flg));
4073     PetscCall(DMPlexConstructGhostCells(dm, flg ? lname : NULL, NULL, &gdm));
4074     PetscCall(DMPlexReplace_Internal(dm, &gdm));
4075   }
4076   /* Handle 1D order */
4077   if (reorder != DMPLEX_REORDER_DEFAULT_FALSE && dim == 1) {
4078     DM           cdm, rdm;
4079     PetscDS      cds;
4080     PetscObject  obj;
4081     PetscClassId id = PETSC_OBJECT_CLASSID;
4082     IS           perm;
4083     PetscInt     Nf;
4084     PetscBool    distributed;
4085 
4086     PetscCall(DMPlexIsDistributed(dm, &distributed));
4087     PetscCall(DMGetCoordinateDM(dm, &cdm));
4088     PetscCall(DMGetDS(cdm, &cds));
4089     PetscCall(PetscDSGetNumFields(cds, &Nf));
4090     if (Nf) {
4091       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
4092       PetscCall(PetscObjectGetClassId(obj, &id));
4093     }
4094     if (!distributed && id != PETSCFE_CLASSID) {
4095       PetscCall(DMPlexGetOrdering1D(dm, &perm));
4096       PetscCall(DMPlexPermute(dm, perm, &rdm));
4097       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4098       PetscCall(ISDestroy(&perm));
4099     }
4100   }
4101   /* Handle */
4102   PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4103   PetscOptionsHeadEnd();
4104   PetscFunctionReturn(0);
4105 }
4106 
4107 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm, Vec *vec)
4108 {
4109   PetscFunctionBegin;
4110   PetscCall(DMCreateGlobalVector_Section_Private(dm, vec));
4111   /* PetscCall(VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM)); */
4112   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex));
4113   PetscCall(VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void))VecView_Plex_Native));
4114   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex));
4115   PetscCall(VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void))VecLoad_Plex_Native));
4116   PetscFunctionReturn(0);
4117 }
4118 
4119 static PetscErrorCode DMCreateLocalVector_Plex(DM dm, Vec *vec)
4120 {
4121   PetscFunctionBegin;
4122   PetscCall(DMCreateLocalVector_Section_Private(dm, vec));
4123   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex_Local));
4124   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex_Local));
4125   PetscFunctionReturn(0);
4126 }
4127 
4128 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
4129 {
4130   PetscInt depth, d;
4131 
4132   PetscFunctionBegin;
4133   PetscCall(DMPlexGetDepth(dm, &depth));
4134   if (depth == 1) {
4135     PetscCall(DMGetDimension(dm, &d));
4136     if (dim == 0) PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4137     else if (dim == d) PetscCall(DMPlexGetDepthStratum(dm, 1, pStart, pEnd));
4138     else {
4139       *pStart = 0;
4140       *pEnd   = 0;
4141     }
4142   } else {
4143     PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4144   }
4145   PetscFunctionReturn(0);
4146 }
4147 
4148 static PetscErrorCode DMGetNeighbors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
4149 {
4150   PetscSF            sf;
4151   PetscInt           niranks, njranks, n;
4152   const PetscMPIInt *iranks, *jranks;
4153   DM_Plex           *data = (DM_Plex *)dm->data;
4154 
4155   PetscFunctionBegin;
4156   PetscCall(DMGetPointSF(dm, &sf));
4157   if (!data->neighbors) {
4158     PetscCall(PetscSFSetUp(sf));
4159     PetscCall(PetscSFGetRootRanks(sf, &njranks, &jranks, NULL, NULL, NULL));
4160     PetscCall(PetscSFGetLeafRanks(sf, &niranks, &iranks, NULL, NULL));
4161     PetscCall(PetscMalloc1(njranks + niranks + 1, &data->neighbors));
4162     PetscCall(PetscArraycpy(data->neighbors + 1, jranks, njranks));
4163     PetscCall(PetscArraycpy(data->neighbors + njranks + 1, iranks, niranks));
4164     n = njranks + niranks;
4165     PetscCall(PetscSortRemoveDupsMPIInt(&n, data->neighbors + 1));
4166     /* The following cast should never fail: can't have more neighbors than PETSC_MPI_INT_MAX */
4167     PetscCall(PetscMPIIntCast(n, data->neighbors));
4168   }
4169   if (nranks) *nranks = data->neighbors[0];
4170   if (ranks) {
4171     if (data->neighbors[0]) *ranks = data->neighbors + 1;
4172     else *ranks = NULL;
4173   }
4174   PetscFunctionReturn(0);
4175 }
4176 
4177 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM, DM, Mat, Vec, Vec);
4178 
4179 static PetscErrorCode DMInitialize_Plex(DM dm)
4180 {
4181   PetscFunctionBegin;
4182   dm->ops->view                      = DMView_Plex;
4183   dm->ops->load                      = DMLoad_Plex;
4184   dm->ops->setfromoptions            = DMSetFromOptions_Plex;
4185   dm->ops->clone                     = DMClone_Plex;
4186   dm->ops->setup                     = DMSetUp_Plex;
4187   dm->ops->createlocalsection        = DMCreateLocalSection_Plex;
4188   dm->ops->createdefaultconstraints  = DMCreateDefaultConstraints_Plex;
4189   dm->ops->createglobalvector        = DMCreateGlobalVector_Plex;
4190   dm->ops->createlocalvector         = DMCreateLocalVector_Plex;
4191   dm->ops->getlocaltoglobalmapping   = NULL;
4192   dm->ops->createfieldis             = NULL;
4193   dm->ops->createcoordinatedm        = DMCreateCoordinateDM_Plex;
4194   dm->ops->createcoordinatefield     = DMCreateCoordinateField_Plex;
4195   dm->ops->getcoloring               = NULL;
4196   dm->ops->creatematrix              = DMCreateMatrix_Plex;
4197   dm->ops->createinterpolation       = DMCreateInterpolation_Plex;
4198   dm->ops->createmassmatrix          = DMCreateMassMatrix_Plex;
4199   dm->ops->createmassmatrixlumped    = DMCreateMassMatrixLumped_Plex;
4200   dm->ops->createinjection           = DMCreateInjection_Plex;
4201   dm->ops->refine                    = DMRefine_Plex;
4202   dm->ops->coarsen                   = DMCoarsen_Plex;
4203   dm->ops->refinehierarchy           = DMRefineHierarchy_Plex;
4204   dm->ops->coarsenhierarchy          = DMCoarsenHierarchy_Plex;
4205   dm->ops->extrude                   = DMExtrude_Plex;
4206   dm->ops->globaltolocalbegin        = NULL;
4207   dm->ops->globaltolocalend          = NULL;
4208   dm->ops->localtoglobalbegin        = NULL;
4209   dm->ops->localtoglobalend          = NULL;
4210   dm->ops->destroy                   = DMDestroy_Plex;
4211   dm->ops->createsubdm               = DMCreateSubDM_Plex;
4212   dm->ops->createsuperdm             = DMCreateSuperDM_Plex;
4213   dm->ops->getdimpoints              = DMGetDimPoints_Plex;
4214   dm->ops->locatepoints              = DMLocatePoints_Plex;
4215   dm->ops->projectfunctionlocal      = DMProjectFunctionLocal_Plex;
4216   dm->ops->projectfunctionlabellocal = DMProjectFunctionLabelLocal_Plex;
4217   dm->ops->projectfieldlocal         = DMProjectFieldLocal_Plex;
4218   dm->ops->projectfieldlabellocal    = DMProjectFieldLabelLocal_Plex;
4219   dm->ops->projectbdfieldlabellocal  = DMProjectBdFieldLabelLocal_Plex;
4220   dm->ops->computel2diff             = DMComputeL2Diff_Plex;
4221   dm->ops->computel2gradientdiff     = DMComputeL2GradientDiff_Plex;
4222   dm->ops->computel2fielddiff        = DMComputeL2FieldDiff_Plex;
4223   dm->ops->getneighbors              = DMGetNeighbors_Plex;
4224   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertBoundaryValues_C", DMPlexInsertBoundaryValues_Plex));
4225   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertTimeDerviativeBoundaryValues_C", DMPlexInsertTimeDerivativeBoundaryValues_Plex));
4226   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMSetUpGLVisViewer_C", DMSetUpGLVisViewer_Plex));
4227   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMCreateNeumannOverlap_C", DMCreateNeumannOverlap_Plex));
4228   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4229   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeGetDefault_C", DMPlexDistributeGetDefault_Plex));
4230   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeSetDefault_C", DMPlexDistributeSetDefault_Plex));
4231   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderGetDefault_C", DMPlexReorderGetDefault_Plex));
4232   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderSetDefault_C", DMPlexReorderSetDefault_Plex));
4233   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMInterpolateSolution_C", DMInterpolateSolution_Plex));
4234   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4235   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetOverlap_C", DMPlexSetOverlap_Plex));
4236   PetscFunctionReturn(0);
4237 }
4238 
4239 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
4240 {
4241   DM_Plex *mesh = (DM_Plex *)dm->data;
4242 
4243   PetscFunctionBegin;
4244   mesh->refct++;
4245   (*newdm)->data = mesh;
4246   PetscCall(PetscObjectChangeTypeName((PetscObject)*newdm, DMPLEX));
4247   PetscCall(DMInitialize_Plex(*newdm));
4248   PetscFunctionReturn(0);
4249 }
4250 
4251 /*MC
4252   DMPLEX = "plex" - A DM object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
4253                     In the local representation, Vecs contain all unknowns in the interior and shared boundary. This is
4254                     specified by a PetscSection object. Ownership in the global representation is determined by
4255                     ownership of the underlying DMPlex points. This is specified by another PetscSection object.
4256 
4257   Options Database Keys:
4258 + -dm_refine_pre                     - Refine mesh before distribution
4259 + -dm_refine_uniform_pre             - Choose uniform or generator-based refinement
4260 + -dm_refine_volume_limit_pre        - Cell volume limit after pre-refinement using generator
4261 . -dm_distribute                     - Distribute mesh across processes
4262 . -dm_distribute_overlap             - Number of cells to overlap for distribution
4263 . -dm_refine                         - Refine mesh after distribution
4264 . -dm_plex_hash_location             - Use grid hashing for point location
4265 . -dm_plex_hash_box_faces <n,m,p>    - The number of divisions in each direction of the grid hash
4266 . -dm_plex_partition_balance         - Attempt to evenly divide points on partition boundary between processes
4267 . -dm_plex_remesh_bd                 - Allow changes to the boundary on remeshing
4268 . -dm_plex_max_projection_height     - Maximum mesh point height used to project locally
4269 . -dm_plex_regular_refinement        - Use special nested projection algorithm for regular refinement
4270 . -dm_plex_check_all                 - Perform all shecks below
4271 . -dm_plex_check_symmetry            - Check that the adjacency information in the mesh is symmetric
4272 . -dm_plex_check_skeleton <celltype> - Check that each cell has the correct number of vertices
4273 . -dm_plex_check_faces <celltype>    - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
4274 . -dm_plex_check_geometry            - Check that cells have positive volume
4275 . -dm_view :mesh.tex:ascii_latex     - View the mesh in LaTeX/TikZ
4276 . -dm_plex_view_scale <num>          - Scale the TikZ
4277 - -dm_plex_print_fem <num>           - View FEM assembly information, such as element vectors and matrices
4278 
4279   Level: intermediate
4280 
4281 .seealso: `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`
4282 M*/
4283 
4284 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
4285 {
4286   DM_Plex *mesh;
4287   PetscInt unit;
4288 
4289   PetscFunctionBegin;
4290   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4291   PetscCall(PetscNew(&mesh));
4292   dm->data = mesh;
4293 
4294   mesh->refct = 1;
4295   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection));
4296   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection));
4297   mesh->refinementUniform      = PETSC_TRUE;
4298   mesh->refinementLimit        = -1.0;
4299   mesh->distDefault            = PETSC_TRUE;
4300   mesh->reorderDefault         = DMPLEX_REORDER_DEFAULT_NOTSET;
4301   mesh->distributionName       = NULL;
4302   mesh->interpolated           = DMPLEX_INTERPOLATED_INVALID;
4303   mesh->interpolatedCollective = DMPLEX_INTERPOLATED_INVALID;
4304 
4305   PetscCall(PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner));
4306   mesh->remeshBd = PETSC_FALSE;
4307 
4308   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
4309 
4310   mesh->depthState    = -1;
4311   mesh->celltypeState = -1;
4312   mesh->printTol      = 1.0e-10;
4313 
4314   PetscCall(DMInitialize_Plex(dm));
4315   PetscFunctionReturn(0);
4316 }
4317 
4318 /*@
4319   DMPlexCreate - Creates a DMPlex object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
4320 
4321   Collective
4322 
4323   Input Parameter:
4324 . comm - The communicator for the DMPlex object
4325 
4326   Output Parameter:
4327 . mesh  - The DMPlex object
4328 
4329   Level: beginner
4330 
4331 @*/
4332 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
4333 {
4334   PetscFunctionBegin;
4335   PetscValidPointer(mesh, 2);
4336   PetscCall(DMCreate(comm, mesh));
4337   PetscCall(DMSetType(*mesh, DMPLEX));
4338   PetscFunctionReturn(0);
4339 }
4340 
4341 /*@C
4342   DMPlexBuildFromCellListParallel - Build distributed DMPLEX topology from a list of vertices for each cell (common mesh generator output)
4343 
4344   Input Parameters:
4345 + dm - The DM
4346 . numCells - The number of cells owned by this process
4347 . numVertices - The number of vertices to be owned by this process, or PETSC_DECIDE
4348 . NVertices - The global number of vertices, or PETSC_DETERMINE
4349 . numCorners - The number of vertices for each cell
4350 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4351 
4352   Output Parameters:
4353 + vertexSF - (Optional) SF describing complete vertex ownership
4354 - verticesAdjSaved - (Optional) vertex adjacency array
4355 
4356   Notes:
4357   Two triangles sharing a face
4358 $
4359 $        2
4360 $      / | \
4361 $     /  |  \
4362 $    /   |   \
4363 $   0  0 | 1  3
4364 $    \   |   /
4365 $     \  |  /
4366 $      \ | /
4367 $        1
4368 would have input
4369 $  numCells = 2, numVertices = 4
4370 $  cells = [0 1 2  1 3 2]
4371 $
4372 which would result in the DMPlex
4373 $
4374 $        4
4375 $      / | \
4376 $     /  |  \
4377 $    /   |   \
4378 $   2  0 | 1  5
4379 $    \   |   /
4380 $     \  |  /
4381 $      \ | /
4382 $        3
4383 
4384   Vertices are implicitly numbered consecutively 0,...,NVertices.
4385   Each rank owns a chunk of numVertices consecutive vertices.
4386   If numVertices is PETSC_DECIDE, PETSc will distribute them as evenly as possible using PetscLayout.
4387   If NVertices is PETSC_DETERMINE and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
4388   If only NVertices is PETSC_DETERMINE, it is computed as the sum of numVertices over all ranks.
4389 
4390   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
4391 
4392   Not currently supported in Fortran.
4393 
4394   Level: advanced
4395 
4396 .seealso: `DMPlexBuildFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildCoordinatesFromCellListParallel()`
4397 @*/
4398 PetscErrorCode DMPlexBuildFromCellListParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, const PetscInt cells[], PetscSF *vertexSF, PetscInt **verticesAdjSaved)
4399 {
4400   PetscSF     sfPoint;
4401   PetscLayout layout;
4402   PetscInt    numVerticesAdj, *verticesAdj, *cones, c, p;
4403 
4404   PetscFunctionBegin;
4405   PetscValidLogicalCollectiveInt(dm, NVertices, 4);
4406   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4407   /* Get/check global number of vertices */
4408   {
4409     PetscInt       NVerticesInCells, i;
4410     const PetscInt len = numCells * numCorners;
4411 
4412     /* NVerticesInCells = max(cells) + 1 */
4413     NVerticesInCells = PETSC_MIN_INT;
4414     for (i = 0; i < len; i++)
4415       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4416     ++NVerticesInCells;
4417     PetscCallMPI(MPI_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
4418 
4419     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
4420     else
4421       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
4422   }
4423   /* Count locally unique vertices */
4424   {
4425     PetscHSetI vhash;
4426     PetscInt   off = 0;
4427 
4428     PetscCall(PetscHSetICreate(&vhash));
4429     for (c = 0; c < numCells; ++c) {
4430       for (p = 0; p < numCorners; ++p) PetscCall(PetscHSetIAdd(vhash, cells[c * numCorners + p]));
4431     }
4432     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
4433     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
4434     else verticesAdj = *verticesAdjSaved;
4435     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
4436     PetscCall(PetscHSetIDestroy(&vhash));
4437     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
4438   }
4439   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
4440   /* Create cones */
4441   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
4442   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
4443   PetscCall(DMSetUp(dm));
4444   PetscCall(DMPlexGetCones(dm, &cones));
4445   for (c = 0; c < numCells; ++c) {
4446     for (p = 0; p < numCorners; ++p) {
4447       const PetscInt gv = cells[c * numCorners + p];
4448       PetscInt       lv;
4449 
4450       /* Positions within verticesAdj form 0-based local vertex numbering;
4451          we need to shift it by numCells to get correct DAG points (cells go first) */
4452       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
4453       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
4454       cones[c * numCorners + p] = lv + numCells;
4455     }
4456   }
4457   /* Build point sf */
4458   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
4459   PetscCall(PetscLayoutSetSize(layout, NVertices));
4460   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
4461   PetscCall(PetscLayoutSetBlockSize(layout, 1));
4462   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
4463   PetscCall(PetscLayoutDestroy(&layout));
4464   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
4465   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
4466   if (dm->sf) {
4467     const char *prefix;
4468 
4469     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
4470     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
4471   }
4472   PetscCall(DMSetPointSF(dm, sfPoint));
4473   PetscCall(PetscSFDestroy(&sfPoint));
4474   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)(*vertexSF), "Vertex Ownership SF"));
4475   /* Fill in the rest of the topology structure */
4476   PetscCall(DMPlexSymmetrize(dm));
4477   PetscCall(DMPlexStratify(dm));
4478   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4479   PetscFunctionReturn(0);
4480 }
4481 
4482 /*@C
4483   DMPlexBuildCoordinatesFromCellListParallel - Build DM coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4484 
4485   Input Parameters:
4486 + dm - The DM
4487 . spaceDim - The spatial dimension used for coordinates
4488 . sfVert - SF describing complete vertex ownership
4489 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4490 
4491   Level: advanced
4492 
4493   Notes:
4494   Not currently supported in Fortran.
4495 
4496 .seealso: `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellListParallel()`
4497 @*/
4498 PetscErrorCode DMPlexBuildCoordinatesFromCellListParallel(DM dm, PetscInt spaceDim, PetscSF sfVert, const PetscReal vertexCoords[])
4499 {
4500   PetscSection coordSection;
4501   Vec          coordinates;
4502   PetscScalar *coords;
4503   PetscInt     numVertices, numVerticesAdj, coordSize, v, vStart, vEnd;
4504 
4505   PetscFunctionBegin;
4506   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4507   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
4508   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4509   PetscCall(DMSetCoordinateDim(dm, spaceDim));
4510   PetscCall(PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL));
4511   PetscCheck(vEnd - vStart == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Supplied sfVert has wrong number of leaves = %" PetscInt_FMT " != %" PetscInt_FMT " = vEnd - vStart", numVerticesAdj, vEnd - vStart);
4512   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4513   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4514   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
4515   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
4516   for (v = vStart; v < vEnd; ++v) {
4517     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
4518     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
4519   }
4520   PetscCall(PetscSectionSetUp(coordSection));
4521   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4522   PetscCall(VecCreate(PetscObjectComm((PetscObject)dm), &coordinates));
4523   PetscCall(VecSetBlockSize(coordinates, spaceDim));
4524   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4525   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4526   PetscCall(VecSetType(coordinates, VECSTANDARD));
4527   PetscCall(VecGetArray(coordinates, &coords));
4528   {
4529     MPI_Datatype coordtype;
4530 
4531     /* Need a temp buffer for coords if we have complex/single */
4532     PetscCallMPI(MPI_Type_contiguous(spaceDim, MPIU_SCALAR, &coordtype));
4533     PetscCallMPI(MPI_Type_commit(&coordtype));
4534 #if defined(PETSC_USE_COMPLEX)
4535     {
4536       PetscScalar *svertexCoords;
4537       PetscInt     i;
4538       PetscCall(PetscMalloc1(numVertices * spaceDim, &svertexCoords));
4539       for (i = 0; i < numVertices * spaceDim; i++) svertexCoords[i] = vertexCoords[i];
4540       PetscCall(PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4541       PetscCall(PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4542       PetscCall(PetscFree(svertexCoords));
4543     }
4544 #else
4545     PetscCall(PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4546     PetscCall(PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4547 #endif
4548     PetscCallMPI(MPI_Type_free(&coordtype));
4549   }
4550   PetscCall(VecRestoreArray(coordinates, &coords));
4551   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4552   PetscCall(VecDestroy(&coordinates));
4553   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4554   PetscFunctionReturn(0);
4555 }
4556 
4557 /*@
4558   DMPlexCreateFromCellListParallelPetsc - Create distributed DMPLEX from a list of vertices for each cell (common mesh generator output)
4559 
4560   Input Parameters:
4561 + comm - The communicator
4562 . dim - The topological dimension of the mesh
4563 . numCells - The number of cells owned by this process
4564 . numVertices - The number of vertices owned by this process, or PETSC_DECIDE
4565 . NVertices - The global number of vertices, or PETSC_DECIDE
4566 . numCorners - The number of vertices for each cell
4567 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4568 . cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4569 . spaceDim - The spatial dimension used for coordinates
4570 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4571 
4572   Output Parameters:
4573 + dm - The DM
4574 . vertexSF - (Optional) SF describing complete vertex ownership
4575 - verticesAdjSaved - (Optional) vertex adjacency array
4576 
4577   Notes:
4578   This function is just a convenient sequence of DMCreate(), DMSetType(), DMSetDimension(),
4579   DMPlexBuildFromCellListParallel(), DMPlexInterpolate(), DMPlexBuildCoordinatesFromCellListParallel()
4580 
4581   See DMPlexBuildFromCellListParallel() for an example and details about the topology-related parameters.
4582   See DMPlexBuildCoordinatesFromCellListParallel() for details about the geometry-related parameters.
4583 
4584   Level: intermediate
4585 
4586 .seealso: `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
4587 @*/
4588 PetscErrorCode DMPlexCreateFromCellListParallelPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, PetscInt **verticesAdj, DM *dm)
4589 {
4590   PetscSF sfVert;
4591 
4592   PetscFunctionBegin;
4593   PetscCall(DMCreate(comm, dm));
4594   PetscCall(DMSetType(*dm, DMPLEX));
4595   PetscValidLogicalCollectiveInt(*dm, dim, 2);
4596   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
4597   PetscCall(DMSetDimension(*dm, dim));
4598   PetscCall(DMPlexBuildFromCellListParallel(*dm, numCells, numVertices, NVertices, numCorners, cells, &sfVert, verticesAdj));
4599   if (interpolate) {
4600     DM idm;
4601 
4602     PetscCall(DMPlexInterpolate(*dm, &idm));
4603     PetscCall(DMDestroy(dm));
4604     *dm = idm;
4605   }
4606   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
4607   if (vertexSF) *vertexSF = sfVert;
4608   else PetscCall(PetscSFDestroy(&sfVert));
4609   PetscFunctionReturn(0);
4610 }
4611 
4612 /*@C
4613   DMPlexBuildFromCellList - Build DMPLEX topology from a list of vertices for each cell (common mesh generator output)
4614 
4615   Input Parameters:
4616 + dm - The DM
4617 . numCells - The number of cells owned by this process
4618 . numVertices - The number of vertices owned by this process, or PETSC_DETERMINE
4619 . numCorners - The number of vertices for each cell
4620 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4621 
4622   Level: advanced
4623 
4624   Notes:
4625   Two triangles sharing a face
4626 $
4627 $        2
4628 $      / | \
4629 $     /  |  \
4630 $    /   |   \
4631 $   0  0 | 1  3
4632 $    \   |   /
4633 $     \  |  /
4634 $      \ | /
4635 $        1
4636 would have input
4637 $  numCells = 2, numVertices = 4
4638 $  cells = [0 1 2  1 3 2]
4639 $
4640 which would result in the DMPlex
4641 $
4642 $        4
4643 $      / | \
4644 $     /  |  \
4645 $    /   |   \
4646 $   2  0 | 1  5
4647 $    \   |   /
4648 $     \  |  /
4649 $      \ | /
4650 $        3
4651 
4652   If numVertices is PETSC_DETERMINE, it is computed by PETSc as the maximum vertex index in cells + 1.
4653 
4654   Not currently supported in Fortran.
4655 
4656 .seealso: `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListPetsc()`
4657 @*/
4658 PetscErrorCode DMPlexBuildFromCellList(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const PetscInt cells[])
4659 {
4660   PetscInt *cones, c, p, dim;
4661 
4662   PetscFunctionBegin;
4663   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4664   PetscCall(DMGetDimension(dm, &dim));
4665   /* Get/check global number of vertices */
4666   {
4667     PetscInt       NVerticesInCells, i;
4668     const PetscInt len = numCells * numCorners;
4669 
4670     /* NVerticesInCells = max(cells) + 1 */
4671     NVerticesInCells = PETSC_MIN_INT;
4672     for (i = 0; i < len; i++)
4673       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4674     ++NVerticesInCells;
4675 
4676     if (numVertices == PETSC_DECIDE) numVertices = NVerticesInCells;
4677     else
4678       PetscCheck(numVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, numVertices, NVerticesInCells);
4679   }
4680   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
4681   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
4682   PetscCall(DMSetUp(dm));
4683   PetscCall(DMPlexGetCones(dm, &cones));
4684   for (c = 0; c < numCells; ++c) {
4685     for (p = 0; p < numCorners; ++p) cones[c * numCorners + p] = cells[c * numCorners + p] + numCells;
4686   }
4687   PetscCall(DMPlexSymmetrize(dm));
4688   PetscCall(DMPlexStratify(dm));
4689   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4690   PetscFunctionReturn(0);
4691 }
4692 
4693 /*@C
4694   DMPlexBuildCoordinatesFromCellList - Build DM coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4695 
4696   Input Parameters:
4697 + dm - The DM
4698 . spaceDim - The spatial dimension used for coordinates
4699 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4700 
4701   Level: advanced
4702 
4703   Notes:
4704   Not currently supported in Fortran.
4705 
4706 .seealso: `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellList()`
4707 @*/
4708 PetscErrorCode DMPlexBuildCoordinatesFromCellList(DM dm, PetscInt spaceDim, const PetscReal vertexCoords[])
4709 {
4710   PetscSection coordSection;
4711   Vec          coordinates;
4712   DM           cdm;
4713   PetscScalar *coords;
4714   PetscInt     v, vStart, vEnd, d;
4715 
4716   PetscFunctionBegin;
4717   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4718   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
4719   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4720   PetscCall(DMSetCoordinateDim(dm, spaceDim));
4721   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4722   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4723   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
4724   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
4725   for (v = vStart; v < vEnd; ++v) {
4726     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
4727     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
4728   }
4729   PetscCall(PetscSectionSetUp(coordSection));
4730 
4731   PetscCall(DMGetCoordinateDM(dm, &cdm));
4732   PetscCall(DMCreateLocalVector(cdm, &coordinates));
4733   PetscCall(VecSetBlockSize(coordinates, spaceDim));
4734   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4735   PetscCall(VecGetArrayWrite(coordinates, &coords));
4736   for (v = 0; v < vEnd - vStart; ++v) {
4737     for (d = 0; d < spaceDim; ++d) coords[v * spaceDim + d] = vertexCoords[v * spaceDim + d];
4738   }
4739   PetscCall(VecRestoreArrayWrite(coordinates, &coords));
4740   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4741   PetscCall(VecDestroy(&coordinates));
4742   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4743   PetscFunctionReturn(0);
4744 }
4745 
4746 /*@
4747   DMPlexCreateFromCellListPetsc - Create DMPLEX from a list of vertices for each cell (common mesh generator output), but only process 0 takes in the input
4748 
4749   Collective on comm
4750 
4751   Input Parameters:
4752 + comm - The communicator
4753 . dim - The topological dimension of the mesh
4754 . numCells - The number of cells, only on process 0
4755 . numVertices - The number of vertices owned by this process, or PETSC_DECIDE, only on process 0
4756 . numCorners - The number of vertices for each cell, only on process 0
4757 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4758 . cells - An array of numCells*numCorners numbers, the vertices for each cell, only on process 0
4759 . spaceDim - The spatial dimension used for coordinates
4760 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex, only on process 0
4761 
4762   Output Parameter:
4763 . dm - The DM, which only has points on process 0
4764 
4765   Notes:
4766   This function is just a convenient sequence of DMCreate(), DMSetType(), DMSetDimension(), DMPlexBuildFromCellList(),
4767   DMPlexInterpolate(), DMPlexBuildCoordinatesFromCellList()
4768 
4769   See DMPlexBuildFromCellList() for an example and details about the topology-related parameters.
4770   See DMPlexBuildCoordinatesFromCellList() for details about the geometry-related parameters.
4771   See DMPlexCreateFromCellListParallelPetsc() for parallel input
4772 
4773   Level: intermediate
4774 
4775 .seealso: `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellList()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
4776 @*/
4777 PetscErrorCode DMPlexCreateFromCellListPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], DM *dm)
4778 {
4779   PetscMPIInt rank;
4780 
4781   PetscFunctionBegin;
4782   PetscCheck(dim, comm, PETSC_ERR_ARG_OUTOFRANGE, "This is not appropriate for 0-dimensional meshes. Consider either creating the DM using DMPlexCreateFromDAG(), by hand, or using DMSwarm.");
4783   PetscCallMPI(MPI_Comm_rank(comm, &rank));
4784   PetscCall(DMCreate(comm, dm));
4785   PetscCall(DMSetType(*dm, DMPLEX));
4786   PetscCall(DMSetDimension(*dm, dim));
4787   if (rank == 0) PetscCall(DMPlexBuildFromCellList(*dm, numCells, numVertices, numCorners, cells));
4788   else PetscCall(DMPlexBuildFromCellList(*dm, 0, 0, 0, NULL));
4789   if (interpolate) {
4790     DM idm;
4791 
4792     PetscCall(DMPlexInterpolate(*dm, &idm));
4793     PetscCall(DMDestroy(dm));
4794     *dm = idm;
4795   }
4796   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, vertexCoords));
4797   else PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, NULL));
4798   PetscFunctionReturn(0);
4799 }
4800 
4801 /*@
4802   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a DM
4803 
4804   Input Parameters:
4805 + dm - The empty DM object, usually from DMCreate() and DMSetDimension()
4806 . depth - The depth of the DAG
4807 . numPoints - Array of size depth + 1 containing the number of points at each depth
4808 . coneSize - The cone size of each point
4809 . cones - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
4810 . coneOrientations - The orientation of each cone point
4811 - vertexCoords - An array of numPoints[0]*spacedim numbers representing the coordinates of each vertex, with spacedim the value set via DMSetCoordinateDim()
4812 
4813   Output Parameter:
4814 . dm - The DM
4815 
4816   Note: Two triangles sharing a face would have input
4817 $  depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
4818 $  cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
4819 $ vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
4820 $
4821 which would result in the DMPlex
4822 $
4823 $        4
4824 $      / | \
4825 $     /  |  \
4826 $    /   |   \
4827 $   2  0 | 1  5
4828 $    \   |   /
4829 $     \  |  /
4830 $      \ | /
4831 $        3
4832 $
4833 $ Notice that all points are numbered consecutively, unlike DMPlexCreateFromCellListPetsc()
4834 
4835   Level: advanced
4836 
4837 .seealso: `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
4838 @*/
4839 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
4840 {
4841   Vec          coordinates;
4842   PetscSection coordSection;
4843   PetscScalar *coords;
4844   PetscInt     coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
4845 
4846   PetscFunctionBegin;
4847   PetscCall(DMGetDimension(dm, &dim));
4848   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
4849   PetscCheck(dimEmbed >= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Embedding dimension %" PetscInt_FMT " cannot be less than intrinsic dimension %" PetscInt_FMT, dimEmbed, dim);
4850   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
4851   PetscCall(DMPlexSetChart(dm, pStart, pEnd));
4852   for (p = pStart; p < pEnd; ++p) {
4853     PetscCall(DMPlexSetConeSize(dm, p, coneSize[p - pStart]));
4854     if (firstVertex < 0 && !coneSize[p - pStart]) firstVertex = p - pStart;
4855   }
4856   PetscCheck(firstVertex >= 0 || !numPoints[0], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Expected %" PetscInt_FMT " vertices but could not find any", numPoints[0]);
4857   PetscCall(DMSetUp(dm)); /* Allocate space for cones */
4858   for (p = pStart, off = 0; p < pEnd; off += coneSize[p - pStart], ++p) {
4859     PetscCall(DMPlexSetCone(dm, p, &cones[off]));
4860     PetscCall(DMPlexSetConeOrientation(dm, p, &coneOrientations[off]));
4861   }
4862   PetscCall(DMPlexSymmetrize(dm));
4863   PetscCall(DMPlexStratify(dm));
4864   /* Build coordinates */
4865   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4866   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4867   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dimEmbed));
4868   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numPoints[0]));
4869   for (v = firstVertex; v < firstVertex + numPoints[0]; ++v) {
4870     PetscCall(PetscSectionSetDof(coordSection, v, dimEmbed));
4871     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed));
4872   }
4873   PetscCall(PetscSectionSetUp(coordSection));
4874   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4875   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
4876   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4877   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4878   PetscCall(VecSetBlockSize(coordinates, dimEmbed));
4879   PetscCall(VecSetType(coordinates, VECSTANDARD));
4880   if (vertexCoords) {
4881     PetscCall(VecGetArray(coordinates, &coords));
4882     for (v = 0; v < numPoints[0]; ++v) {
4883       PetscInt off;
4884 
4885       PetscCall(PetscSectionGetOffset(coordSection, v + firstVertex, &off));
4886       for (d = 0; d < dimEmbed; ++d) coords[off + d] = vertexCoords[v * dimEmbed + d];
4887     }
4888   }
4889   PetscCall(VecRestoreArray(coordinates, &coords));
4890   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4891   PetscCall(VecDestroy(&coordinates));
4892   PetscFunctionReturn(0);
4893 }
4894 
4895 /*@C
4896   DMPlexCreateCellVertexFromFile - Create a DMPlex mesh from a simple cell-vertex file.
4897 
4898 + comm        - The MPI communicator
4899 . filename    - Name of the .dat file
4900 - interpolate - Create faces and edges in the mesh
4901 
4902   Output Parameter:
4903 . dm  - The DM object representing the mesh
4904 
4905   Note: The format is the simplest possible:
4906 $ Ne
4907 $ v0 v1 ... vk
4908 $ Nv
4909 $ x y z marker
4910 
4911   Level: beginner
4912 
4913 .seealso: `DMPlexCreateFromFile()`, `DMPlexCreateMedFromFile()`, `DMPlexCreateGmsh()`, `DMPlexCreate()`
4914 @*/
4915 PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
4916 {
4917   DMLabel      marker;
4918   PetscViewer  viewer;
4919   Vec          coordinates;
4920   PetscSection coordSection;
4921   PetscScalar *coords;
4922   char         line[PETSC_MAX_PATH_LEN];
4923   PetscInt     dim = 3, cdim = 3, coordSize, v, c, d;
4924   PetscMPIInt  rank;
4925   int          snum, Nv, Nc, Ncn, Nl;
4926 
4927   PetscFunctionBegin;
4928   PetscCallMPI(MPI_Comm_rank(comm, &rank));
4929   PetscCall(PetscViewerCreate(comm, &viewer));
4930   PetscCall(PetscViewerSetType(viewer, PETSCVIEWERASCII));
4931   PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
4932   PetscCall(PetscViewerFileSetName(viewer, filename));
4933   if (rank == 0) {
4934     PetscCall(PetscViewerRead(viewer, line, 4, NULL, PETSC_STRING));
4935     snum = sscanf(line, "%d %d %d %d", &Nc, &Nv, &Ncn, &Nl);
4936     PetscCheck(snum == 4, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4937   } else {
4938     Nc = Nv = Ncn = Nl = 0;
4939   }
4940   PetscCall(DMCreate(comm, dm));
4941   PetscCall(DMSetType(*dm, DMPLEX));
4942   PetscCall(DMPlexSetChart(*dm, 0, Nc + Nv));
4943   PetscCall(DMSetDimension(*dm, dim));
4944   PetscCall(DMSetCoordinateDim(*dm, cdim));
4945   /* Read topology */
4946   if (rank == 0) {
4947     char     format[PETSC_MAX_PATH_LEN];
4948     PetscInt cone[8];
4949     int      vbuf[8], v;
4950 
4951     for (c = 0; c < Ncn; ++c) {
4952       format[c * 3 + 0] = '%';
4953       format[c * 3 + 1] = 'd';
4954       format[c * 3 + 2] = ' ';
4955     }
4956     format[Ncn * 3 - 1] = '\0';
4957     for (c = 0; c < Nc; ++c) PetscCall(DMPlexSetConeSize(*dm, c, Ncn));
4958     PetscCall(DMSetUp(*dm));
4959     for (c = 0; c < Nc; ++c) {
4960       PetscCall(PetscViewerRead(viewer, line, Ncn, NULL, PETSC_STRING));
4961       switch (Ncn) {
4962       case 2:
4963         snum = sscanf(line, format, &vbuf[0], &vbuf[1]);
4964         break;
4965       case 3:
4966         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2]);
4967         break;
4968       case 4:
4969         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3]);
4970         break;
4971       case 6:
4972         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5]);
4973         break;
4974       case 8:
4975         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);
4976         break;
4977       default:
4978         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No cell shape with %d vertices", Ncn);
4979       }
4980       PetscCheck(snum == Ncn, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4981       for (v = 0; v < Ncn; ++v) cone[v] = vbuf[v] + Nc;
4982       /* Hexahedra are inverted */
4983       if (Ncn == 8) {
4984         PetscInt tmp = cone[1];
4985         cone[1]      = cone[3];
4986         cone[3]      = tmp;
4987       }
4988       PetscCall(DMPlexSetCone(*dm, c, cone));
4989     }
4990   }
4991   PetscCall(DMPlexSymmetrize(*dm));
4992   PetscCall(DMPlexStratify(*dm));
4993   /* Read coordinates */
4994   PetscCall(DMGetCoordinateSection(*dm, &coordSection));
4995   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4996   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
4997   PetscCall(PetscSectionSetChart(coordSection, Nc, Nc + Nv));
4998   for (v = Nc; v < Nc + Nv; ++v) {
4999     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
5000     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
5001   }
5002   PetscCall(PetscSectionSetUp(coordSection));
5003   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
5004   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
5005   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5006   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
5007   PetscCall(VecSetBlockSize(coordinates, cdim));
5008   PetscCall(VecSetType(coordinates, VECSTANDARD));
5009   PetscCall(VecGetArray(coordinates, &coords));
5010   if (rank == 0) {
5011     char   format[PETSC_MAX_PATH_LEN];
5012     double x[3];
5013     int    l, val[3];
5014 
5015     if (Nl) {
5016       for (l = 0; l < Nl; ++l) {
5017         format[l * 3 + 0] = '%';
5018         format[l * 3 + 1] = 'd';
5019         format[l * 3 + 2] = ' ';
5020       }
5021       format[Nl * 3 - 1] = '\0';
5022       PetscCall(DMCreateLabel(*dm, "marker"));
5023       PetscCall(DMGetLabel(*dm, "marker", &marker));
5024     }
5025     for (v = 0; v < Nv; ++v) {
5026       PetscCall(PetscViewerRead(viewer, line, 3 + Nl, NULL, PETSC_STRING));
5027       snum = sscanf(line, "%lg %lg %lg", &x[0], &x[1], &x[2]);
5028       PetscCheck(snum == 3, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5029       switch (Nl) {
5030       case 0:
5031         snum = 0;
5032         break;
5033       case 1:
5034         snum = sscanf(line, format, &val[0]);
5035         break;
5036       case 2:
5037         snum = sscanf(line, format, &val[0], &val[1]);
5038         break;
5039       case 3:
5040         snum = sscanf(line, format, &val[0], &val[1], &val[2]);
5041         break;
5042       default:
5043         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Request support for %d labels", Nl);
5044       }
5045       PetscCheck(snum == Nl, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5046       for (d = 0; d < cdim; ++d) coords[v * cdim + d] = x[d];
5047       for (l = 0; l < Nl; ++l) PetscCall(DMLabelSetValue(marker, v + Nc, val[l]));
5048     }
5049   }
5050   PetscCall(VecRestoreArray(coordinates, &coords));
5051   PetscCall(DMSetCoordinatesLocal(*dm, coordinates));
5052   PetscCall(VecDestroy(&coordinates));
5053   PetscCall(PetscViewerDestroy(&viewer));
5054   if (interpolate) {
5055     DM      idm;
5056     DMLabel bdlabel;
5057 
5058     PetscCall(DMPlexInterpolate(*dm, &idm));
5059     PetscCall(DMDestroy(dm));
5060     *dm = idm;
5061 
5062     if (!Nl) {
5063       PetscCall(DMCreateLabel(*dm, "marker"));
5064       PetscCall(DMGetLabel(*dm, "marker", &bdlabel));
5065       PetscCall(DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel));
5066       PetscCall(DMPlexLabelComplete(*dm, bdlabel));
5067     }
5068   }
5069   PetscFunctionReturn(0);
5070 }
5071 
5072 /*@C
5073   DMPlexCreateFromFile - This takes a filename and produces a DM
5074 
5075   Input Parameters:
5076 + comm - The communicator
5077 . filename - A file name
5078 . plexname - The object name of the resulting DM, also used for intra-datafile lookup by some formats
5079 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
5080 
5081   Output Parameter:
5082 . dm - The DM
5083 
5084   Options Database Keys:
5085 . -dm_plex_create_from_hdf5_xdmf - use the PETSC_VIEWER_HDF5_XDMF format for reading HDF5
5086 
5087   Use -dm_plex_create_ prefix to pass options to the internal PetscViewer, e.g.
5088 $ -dm_plex_create_viewer_hdf5_collective
5089 
5090   Notes:
5091   Using PETSCVIEWERHDF5 type with PETSC_VIEWER_HDF5_PETSC format, one can save multiple DMPlex
5092   meshes in a single HDF5 file. This in turn requires one to name the DMPlex object with PetscObjectSetName()
5093   before saving it with DMView() and before loading it with DMLoad() for identification of the mesh object.
5094   The input parameter name is thus used to name the DMPlex object when DMPlexCreateFromFile() internally
5095   calls DMLoad(). Currently, name is ignored for other viewer types and/or formats.
5096 
5097   Level: beginner
5098 
5099 .seealso: `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`, `PetscObjectSetName()`, `DMView()`, `DMLoad()`
5100 @*/
5101 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], const char plexname[], PetscBool interpolate, DM *dm)
5102 {
5103   const char  extGmsh[]      = ".msh";
5104   const char  extGmsh2[]     = ".msh2";
5105   const char  extGmsh4[]     = ".msh4";
5106   const char  extCGNS[]      = ".cgns";
5107   const char  extExodus[]    = ".exo";
5108   const char  extExodus_e[]  = ".e";
5109   const char  extGenesis[]   = ".gen";
5110   const char  extFluent[]    = ".cas";
5111   const char  extHDF5[]      = ".h5";
5112   const char  extMed[]       = ".med";
5113   const char  extPLY[]       = ".ply";
5114   const char  extEGADSLite[] = ".egadslite";
5115   const char  extEGADS[]     = ".egads";
5116   const char  extIGES[]      = ".igs";
5117   const char  extSTEP[]      = ".stp";
5118   const char  extCV[]        = ".dat";
5119   size_t      len;
5120   PetscBool   isGmsh, isGmsh2, isGmsh4, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isMed, isPLY, isEGADSLite, isEGADS, isIGES, isSTEP, isCV;
5121   PetscMPIInt rank;
5122 
5123   PetscFunctionBegin;
5124   PetscValidCharPointer(filename, 2);
5125   if (plexname) PetscValidCharPointer(plexname, 3);
5126   PetscValidPointer(dm, 5);
5127   PetscCall(DMInitializePackage());
5128   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5129   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5130   PetscCall(PetscStrlen(filename, &len));
5131   PetscCheck(len, comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
5132 
5133 #define CheckExtension(extension__, is_extension__) \
5134   do { \
5135     PetscAssert(sizeof(extension__), comm, PETSC_ERR_PLIB, "Zero-size extension: %s", extension__); \
5136     /* don't count the null-terminator at the end */ \
5137     const size_t ext_len = sizeof(extension__) - 1; \
5138     if (len < ext_len) { \
5139       is_extension__ = PETSC_FALSE; \
5140     } else { \
5141       PetscCall(PetscStrncmp(filename + len - ext_len, extension__, ext_len, &is_extension__)); \
5142     } \
5143   } while (0)
5144 
5145   CheckExtension(extGmsh, isGmsh);
5146   CheckExtension(extGmsh2, isGmsh2);
5147   CheckExtension(extGmsh4, isGmsh4);
5148   CheckExtension(extCGNS, isCGNS);
5149   CheckExtension(extExodus, isExodus);
5150   if (!isExodus) CheckExtension(extExodus_e, isExodus);
5151   CheckExtension(extGenesis, isGenesis);
5152   CheckExtension(extFluent, isFluent);
5153   CheckExtension(extHDF5, isHDF5);
5154   CheckExtension(extMed, isMed);
5155   CheckExtension(extPLY, isPLY);
5156   CheckExtension(extEGADSLite, isEGADSLite);
5157   CheckExtension(extEGADS, isEGADS);
5158   CheckExtension(extIGES, isIGES);
5159   CheckExtension(extSTEP, isSTEP);
5160   CheckExtension(extCV, isCV);
5161 
5162 #undef CheckExtension
5163 
5164   if (isGmsh || isGmsh2 || isGmsh4) {
5165     PetscCall(DMPlexCreateGmshFromFile(comm, filename, interpolate, dm));
5166   } else if (isCGNS) {
5167     PetscCall(DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm));
5168   } else if (isExodus || isGenesis) {
5169     PetscCall(DMPlexCreateExodusFromFile(comm, filename, interpolate, dm));
5170   } else if (isFluent) {
5171     PetscCall(DMPlexCreateFluentFromFile(comm, filename, interpolate, dm));
5172   } else if (isHDF5) {
5173     PetscBool   load_hdf5_xdmf = PETSC_FALSE;
5174     PetscViewer viewer;
5175 
5176     /* PETSC_VIEWER_HDF5_XDMF is used if the filename ends with .xdmf.h5, or if -dm_plex_create_from_hdf5_xdmf option is present */
5177     PetscCall(PetscStrncmp(&filename[PetscMax(0, len - 8)], ".xdmf", 5, &load_hdf5_xdmf));
5178     PetscCall(PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &load_hdf5_xdmf, NULL));
5179     PetscCall(PetscViewerCreate(comm, &viewer));
5180     PetscCall(PetscViewerSetType(viewer, PETSCVIEWERHDF5));
5181     PetscCall(PetscViewerSetOptionsPrefix(viewer, "dm_plex_create_"));
5182     PetscCall(PetscViewerSetFromOptions(viewer));
5183     PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
5184     PetscCall(PetscViewerFileSetName(viewer, filename));
5185 
5186     PetscCall(DMCreate(comm, dm));
5187     PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5188     PetscCall(DMSetType(*dm, DMPLEX));
5189     if (load_hdf5_xdmf) PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF));
5190     PetscCall(DMLoad(*dm, viewer));
5191     if (load_hdf5_xdmf) PetscCall(PetscViewerPopFormat(viewer));
5192     PetscCall(PetscViewerDestroy(&viewer));
5193 
5194     if (interpolate) {
5195       DM idm;
5196 
5197       PetscCall(DMPlexInterpolate(*dm, &idm));
5198       PetscCall(DMDestroy(dm));
5199       *dm = idm;
5200     }
5201   } else if (isMed) {
5202     PetscCall(DMPlexCreateMedFromFile(comm, filename, interpolate, dm));
5203   } else if (isPLY) {
5204     PetscCall(DMPlexCreatePLYFromFile(comm, filename, interpolate, dm));
5205   } else if (isEGADSLite || isEGADS || isIGES || isSTEP) {
5206     if (isEGADSLite) PetscCall(DMPlexCreateEGADSLiteFromFile(comm, filename, dm));
5207     else PetscCall(DMPlexCreateEGADSFromFile(comm, filename, dm));
5208     if (!interpolate) {
5209       DM udm;
5210 
5211       PetscCall(DMPlexUninterpolate(*dm, &udm));
5212       PetscCall(DMDestroy(dm));
5213       *dm = udm;
5214     }
5215   } else if (isCV) {
5216     PetscCall(DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm));
5217   } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
5218   PetscCall(PetscStrlen(plexname, &len));
5219   if (len) PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5220   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5221   PetscFunctionReturn(0);
5222 }
5223 /*@C
5224   DMPlexCreateEphemeral - This takes a `DMPlexTransform` and a base `DMPlex` and produces an ephemeral `DM`, meaning one that is created on the fly in response to queries.
5225 
5226   Input Parameter:
5227 . tr - The `DMPlexTransform`
5228 
5229   Output Parameter:
5230 . dm - The `DM`
5231 
5232   Notes:
5233   An emphemeral mesh is one that is not stored concretely, as in the default Plex implementation, but rather is produced on the fly in response to queries, using information from the transform and the base mesh.
5234 
5235   Level: beginner
5236 
5237 .seealso: `DMPlexCreateFromFile`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
5238 @*/
5239 PetscErrorCode DMPlexCreateEphemeral(DMPlexTransform tr, DM *dm)
5240 {
5241   DM       bdm;
5242   PetscInt Nl;
5243 
5244   PetscFunctionBegin;
5245   PetscCall(DMCreate(PetscObjectComm((PetscObject)tr), dm));
5246   PetscCall(DMSetType(*dm, DMPLEX));
5247   PetscCall(DMSetFromOptions(*dm));
5248 
5249   PetscCall(PetscObjectReference((PetscObject)tr));
5250   PetscCall(DMPlexTransformDestroy(&((DM_Plex *)(*dm)->data)->tr));
5251   ((DM_Plex *)(*dm)->data)->tr = tr;
5252 
5253   PetscCall(DMPlexTransformGetDM(tr, &bdm));
5254   PetscCall(DMGetNumLabels(bdm, &Nl));
5255   for (PetscInt l = 0; l < Nl; ++l) {
5256     DMLabel     label, labelNew;
5257     const char *lname;
5258     PetscBool   isDepth, isCellType;
5259 
5260     PetscCall(DMGetLabelName(bdm, l, &lname));
5261     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
5262     if (isDepth) continue;
5263     PetscCall(PetscStrcmp(lname, "celltype", &isCellType));
5264     if (isCellType) continue;
5265     PetscCall(DMCreateLabel(*dm, lname));
5266     PetscCall(DMGetLabel(bdm, lname, &label));
5267     PetscCall(DMGetLabel(*dm, lname, &labelNew));
5268     PetscCall(DMLabelSetType(labelNew, DMLABELEPHEMERAL));
5269     PetscCall(DMLabelEphemeralSetLabel(labelNew, label));
5270     PetscCall(DMLabelEphemeralSetTransform(labelNew, tr));
5271     PetscCall(DMLabelSetUp(labelNew));
5272   }
5273   PetscFunctionReturn(0);
5274 }
5275