xref: /petsc/src/dm/impls/plex/plexcreate.c (revision dd4c3f67496275a10180fa3e9c7ac555daec1908)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h> /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>   /*I   "petscdmplex.h"   I*/
4 #include <petscsf.h>
5 #include <petscdmplextransform.h>
6 #include <petscdmlabelephemeral.h>
7 #include <petsc/private/kernels/blockmatmult.h>
8 #include <petsc/private/kernels/blockinvert.h>
9 
10 PetscLogEvent DMPLEX_CreateFromFile, DMPLEX_BuildFromCellList, DMPLEX_BuildCoordinatesFromCellList;
11 
12 /* External function declarations here */
13 static PetscErrorCode DMInitialize_Plex(DM dm);
14 
15 /* This copies internal things in the Plex structure that we generally want when making a new, related Plex */
16 PetscErrorCode DMPlexCopy_Internal(DM dmin, PetscBool copyPeriodicity, PetscBool copyOverlap, DM dmout)
17 {
18   const PetscReal         *maxCell, *Lstart, *L;
19   PetscBool                dist;
20   DMPlexReorderDefaultFlag reorder;
21 
22   PetscFunctionBegin;
23   if (copyPeriodicity) {
24     PetscCall(DMGetPeriodicity(dmin, &maxCell, &Lstart, &L));
25     PetscCall(DMSetPeriodicity(dmout, maxCell, Lstart, L));
26   }
27   PetscCall(DMPlexDistributeGetDefault(dmin, &dist));
28   PetscCall(DMPlexDistributeSetDefault(dmout, dist));
29   PetscCall(DMPlexReorderGetDefault(dmin, &reorder));
30   PetscCall(DMPlexReorderSetDefault(dmout, reorder));
31   ((DM_Plex *)dmout->data)->useHashLocation = ((DM_Plex *)dmin->data)->useHashLocation;
32   if (copyOverlap) PetscCall(DMPlexSetOverlap_Plex(dmout, dmin, 0));
33   PetscFunctionReturn(0);
34 }
35 
36 /* Replace dm with the contents of ndm, and then destroy ndm
37    - Share the DM_Plex structure
38    - Share the coordinates
39    - Share the SF
40 */
41 PetscErrorCode DMPlexReplace_Internal(DM dm, DM *ndm)
42 {
43   PetscSF          sf;
44   DM               dmNew = *ndm, coordDM, coarseDM;
45   Vec              coords;
46   const PetscReal *maxCell, *Lstart, *L;
47   PetscInt         dim, cdim;
48 
49   PetscFunctionBegin;
50   if (dm == dmNew) {
51     PetscCall(DMDestroy(ndm));
52     PetscFunctionReturn(0);
53   }
54   dm->setupcalled = dmNew->setupcalled;
55   PetscCall(DMGetDimension(dmNew, &dim));
56   PetscCall(DMSetDimension(dm, dim));
57   PetscCall(DMGetCoordinateDim(dmNew, &cdim));
58   PetscCall(DMSetCoordinateDim(dm, cdim));
59   PetscCall(DMGetPointSF(dmNew, &sf));
60   PetscCall(DMSetPointSF(dm, sf));
61   PetscCall(DMGetCoordinateDM(dmNew, &coordDM));
62   PetscCall(DMGetCoordinatesLocal(dmNew, &coords));
63   PetscCall(DMSetCoordinateDM(dm, coordDM));
64   PetscCall(DMSetCoordinatesLocal(dm, coords));
65   PetscCall(DMGetCellCoordinateDM(dmNew, &coordDM));
66   PetscCall(DMGetCellCoordinatesLocal(dmNew, &coords));
67   PetscCall(DMSetCellCoordinateDM(dm, coordDM));
68   PetscCall(DMSetCellCoordinatesLocal(dm, coords));
69   /* Do not want to create the coordinate field if it does not already exist, so do not call DMGetCoordinateField() */
70   PetscCall(DMFieldDestroy(&dm->coordinates[0].field));
71   dm->coordinates[0].field            = dmNew->coordinates[0].field;
72   ((DM_Plex *)dmNew->data)->coordFunc = ((DM_Plex *)dm->data)->coordFunc;
73   PetscCall(DMGetPeriodicity(dmNew, &maxCell, &Lstart, &L));
74   PetscCall(DMSetPeriodicity(dm, maxCell, Lstart, L));
75   PetscCall(DMDestroy_Plex(dm));
76   PetscCall(DMInitialize_Plex(dm));
77   dm->data = dmNew->data;
78   ((DM_Plex *)dmNew->data)->refct++;
79   PetscCall(DMDestroyLabelLinkList_Internal(dm));
80   PetscCall(DMCopyLabels(dmNew, dm, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL));
81   PetscCall(DMGetCoarseDM(dmNew, &coarseDM));
82   PetscCall(DMSetCoarseDM(dm, coarseDM));
83   PetscCall(DMDestroy(ndm));
84   PetscFunctionReturn(0);
85 }
86 
87 /* Swap dm with the contents of dmNew
88    - Swap the DM_Plex structure
89    - Swap the coordinates
90    - Swap the point PetscSF
91 */
92 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
93 {
94   DM          coordDMA, coordDMB;
95   Vec         coordsA, coordsB;
96   PetscSF     sfA, sfB;
97   DMField     fieldTmp;
98   void       *tmp;
99   DMLabelLink listTmp;
100   DMLabel     depthTmp;
101   PetscInt    tmpI;
102 
103   PetscFunctionBegin;
104   if (dmA == dmB) PetscFunctionReturn(0);
105   PetscCall(DMGetPointSF(dmA, &sfA));
106   PetscCall(DMGetPointSF(dmB, &sfB));
107   PetscCall(PetscObjectReference((PetscObject)sfA));
108   PetscCall(DMSetPointSF(dmA, sfB));
109   PetscCall(DMSetPointSF(dmB, sfA));
110   PetscCall(PetscObjectDereference((PetscObject)sfA));
111 
112   PetscCall(DMGetCoordinateDM(dmA, &coordDMA));
113   PetscCall(DMGetCoordinateDM(dmB, &coordDMB));
114   PetscCall(PetscObjectReference((PetscObject)coordDMA));
115   PetscCall(DMSetCoordinateDM(dmA, coordDMB));
116   PetscCall(DMSetCoordinateDM(dmB, coordDMA));
117   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
118 
119   PetscCall(DMGetCoordinatesLocal(dmA, &coordsA));
120   PetscCall(DMGetCoordinatesLocal(dmB, &coordsB));
121   PetscCall(PetscObjectReference((PetscObject)coordsA));
122   PetscCall(DMSetCoordinatesLocal(dmA, coordsB));
123   PetscCall(DMSetCoordinatesLocal(dmB, coordsA));
124   PetscCall(PetscObjectDereference((PetscObject)coordsA));
125 
126   PetscCall(DMGetCellCoordinateDM(dmA, &coordDMA));
127   PetscCall(DMGetCellCoordinateDM(dmB, &coordDMB));
128   PetscCall(PetscObjectReference((PetscObject)coordDMA));
129   PetscCall(DMSetCellCoordinateDM(dmA, coordDMB));
130   PetscCall(DMSetCellCoordinateDM(dmB, coordDMA));
131   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
132 
133   PetscCall(DMGetCellCoordinatesLocal(dmA, &coordsA));
134   PetscCall(DMGetCellCoordinatesLocal(dmB, &coordsB));
135   PetscCall(PetscObjectReference((PetscObject)coordsA));
136   PetscCall(DMSetCellCoordinatesLocal(dmA, coordsB));
137   PetscCall(DMSetCellCoordinatesLocal(dmB, coordsA));
138   PetscCall(PetscObjectDereference((PetscObject)coordsA));
139 
140   fieldTmp                  = dmA->coordinates[0].field;
141   dmA->coordinates[0].field = dmB->coordinates[0].field;
142   dmB->coordinates[0].field = fieldTmp;
143   fieldTmp                  = dmA->coordinates[1].field;
144   dmA->coordinates[1].field = dmB->coordinates[1].field;
145   dmB->coordinates[1].field = fieldTmp;
146   tmp                       = dmA->data;
147   dmA->data                 = dmB->data;
148   dmB->data                 = tmp;
149   listTmp                   = dmA->labels;
150   dmA->labels               = dmB->labels;
151   dmB->labels               = listTmp;
152   depthTmp                  = dmA->depthLabel;
153   dmA->depthLabel           = dmB->depthLabel;
154   dmB->depthLabel           = depthTmp;
155   depthTmp                  = dmA->celltypeLabel;
156   dmA->celltypeLabel        = dmB->celltypeLabel;
157   dmB->celltypeLabel        = depthTmp;
158   tmpI                      = dmA->levelup;
159   dmA->levelup              = dmB->levelup;
160   dmB->levelup              = tmpI;
161   PetscFunctionReturn(0);
162 }
163 
164 static PetscErrorCode DMPlexInterpolateInPlace_Internal(DM dm)
165 {
166   DM idm;
167 
168   PetscFunctionBegin;
169   PetscCall(DMPlexInterpolate(dm, &idm));
170   PetscCall(DMPlexCopyCoordinates(dm, idm));
171   PetscCall(DMPlexReplace_Internal(dm, &idm));
172   PetscFunctionReturn(0);
173 }
174 
175 /*@C
176   DMPlexCreateCoordinateSpace - Creates a finite element space for the coordinates
177 
178   Collective
179 
180   Input Parameters:
181 + DM        - The DM
182 . degree    - The degree of the finite element or PETSC_DECIDE
183 - coordFunc - An optional function to map new points from refinement to the surface
184 
185   Level: advanced
186 
187 .seealso: `PetscFECreateLagrange()`, `DMGetCoordinateDM()`
188 @*/
189 PetscErrorCode DMPlexCreateCoordinateSpace(DM dm, PetscInt degree, PetscPointFunc coordFunc)
190 {
191   DM_Plex     *mesh = (DM_Plex *)dm->data;
192   DM           cdm;
193   PetscDS      cds;
194   PetscFE      fe;
195   PetscClassId id;
196 
197   PetscFunctionBegin;
198   PetscCall(DMGetCoordinateDM(dm, &cdm));
199   PetscCall(DMGetDS(cdm, &cds));
200   PetscCall(PetscDSGetDiscretization(cds, 0, (PetscObject *)&fe));
201   PetscCall(PetscObjectGetClassId((PetscObject)fe, &id));
202   if (id != PETSCFE_CLASSID) {
203     PetscBool simplex;
204     PetscInt dim, dE, qorder;
205 
206     PetscCall(DMGetDimension(dm, &dim));
207     PetscCall(DMGetCoordinateDim(dm, &dE));
208     qorder = degree;
209     PetscObjectOptionsBegin((PetscObject)cdm);
210     PetscCall(PetscOptionsBoundedInt("-coord_dm_default_quadrature_order", "Quadrature order is one less than quadrature points per edge", "DMPlexCreateCoordinateSpace", qorder, &qorder, NULL, 0));
211     PetscOptionsEnd();
212     if (degree == PETSC_DECIDE) fe = NULL;
213     else {
214       PetscCall(DMPlexIsSimplex(dm, &simplex));
215       PetscCall(PetscFECreateLagrange(PETSC_COMM_SELF, dim, dE, simplex, degree, qorder, &fe));
216     }
217     PetscCall(DMProjectCoordinates(dm, fe));
218     PetscCall(PetscFEDestroy(&fe));
219   }
220   mesh->coordFunc = coordFunc;
221   PetscFunctionReturn(0);
222 }
223 
224 /*@
225   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
226 
227   Collective
228 
229   Input Parameters:
230 + comm - The communicator for the DM object
231 . dim - The spatial dimension
232 . simplex - Flag for simplicial cells, otherwise they are tensor product cells
233 . interpolate - Flag to create intermediate mesh pieces (edges, faces)
234 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
235 
236   Output Parameter:
237 . dm - The DM object
238 
239   Level: beginner
240 
241 .seealso: `DMSetType()`, `DMCreate()`
242 @*/
243 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscReal refinementLimit, DM *newdm)
244 {
245   DM          dm;
246   PetscMPIInt rank;
247 
248   PetscFunctionBegin;
249   PetscCall(DMCreate(comm, &dm));
250   PetscCall(DMSetType(dm, DMPLEX));
251   PetscCall(DMSetDimension(dm, dim));
252   PetscCallMPI(MPI_Comm_rank(comm, &rank));
253   switch (dim) {
254   case 2:
255     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "triangular"));
256     else PetscCall(PetscObjectSetName((PetscObject)dm, "quadrilateral"));
257     break;
258   case 3:
259     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "tetrahedral"));
260     else PetscCall(PetscObjectSetName((PetscObject)dm, "hexahedral"));
261     break;
262   default:
263     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
264   }
265   if (rank) {
266     PetscInt numPoints[2] = {0, 0};
267     PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL));
268   } else {
269     switch (dim) {
270     case 2:
271       if (simplex) {
272         PetscInt    numPoints[2]        = {4, 2};
273         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
274         PetscInt    cones[6]            = {2, 3, 4, 5, 4, 3};
275         PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
276         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
277 
278         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
279       } else {
280         PetscInt    numPoints[2]        = {6, 2};
281         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
282         PetscInt    cones[8]            = {2, 3, 4, 5, 3, 6, 7, 4};
283         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
284         PetscScalar vertexCoords[12]    = {-1.0, -0.5, 0.0, -0.5, 0.0, 0.5, -1.0, 0.5, 1.0, -0.5, 1.0, 0.5};
285 
286         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
287       }
288       break;
289     case 3:
290       if (simplex) {
291         PetscInt    numPoints[2]        = {5, 2};
292         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
293         PetscInt    cones[8]            = {4, 3, 5, 2, 5, 3, 4, 6};
294         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
295         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0};
296 
297         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
298       } else {
299         PetscInt    numPoints[2]         = {12, 2};
300         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
301         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9, 5, 4, 10, 11, 7, 12, 13, 8};
302         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
303         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5, -1.0, 0.5, -0.5, 0.0, 0.5, -0.5, 0.0, -0.5, -0.5, -1.0, -0.5, 0.5, 0.0, -0.5, 0.5, 0.0, 0.5, 0.5, -1.0, 0.5, 0.5, 1.0, 0.5, -0.5, 1.0, -0.5, -0.5, 1.0, -0.5, 0.5, 1.0, 0.5, 0.5};
304 
305         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
306       }
307       break;
308     default:
309       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
310     }
311   }
312   *newdm = dm;
313   if (refinementLimit > 0.0) {
314     DM          rdm;
315     const char *name;
316 
317     PetscCall(DMPlexSetRefinementUniform(*newdm, PETSC_FALSE));
318     PetscCall(DMPlexSetRefinementLimit(*newdm, refinementLimit));
319     PetscCall(DMRefine(*newdm, comm, &rdm));
320     PetscCall(PetscObjectGetName((PetscObject)*newdm, &name));
321     PetscCall(PetscObjectSetName((PetscObject)rdm, name));
322     PetscCall(DMDestroy(newdm));
323     *newdm = rdm;
324   }
325   if (interpolate) {
326     DM idm;
327 
328     PetscCall(DMPlexInterpolate(*newdm, &idm));
329     PetscCall(DMDestroy(newdm));
330     *newdm = idm;
331   }
332   PetscFunctionReturn(0);
333 }
334 
335 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
336 {
337   const PetscInt numVertices    = 2;
338   PetscInt       markerRight    = 1;
339   PetscInt       markerLeft     = 1;
340   PetscBool      markerSeparate = PETSC_FALSE;
341   Vec            coordinates;
342   PetscSection   coordSection;
343   PetscScalar   *coords;
344   PetscInt       coordSize;
345   PetscMPIInt    rank;
346   PetscInt       cdim = 1, v;
347 
348   PetscFunctionBegin;
349   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
350   if (markerSeparate) {
351     markerRight = 2;
352     markerLeft  = 1;
353   }
354   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
355   if (rank == 0) {
356     PetscCall(DMPlexSetChart(dm, 0, numVertices));
357     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
358     PetscCall(DMSetLabelValue(dm, "marker", 0, markerLeft));
359     PetscCall(DMSetLabelValue(dm, "marker", 1, markerRight));
360   }
361   PetscCall(DMPlexSymmetrize(dm));
362   PetscCall(DMPlexStratify(dm));
363   /* Build coordinates */
364   PetscCall(DMSetCoordinateDim(dm, cdim));
365   PetscCall(DMGetCoordinateSection(dm, &coordSection));
366   PetscCall(PetscSectionSetNumFields(coordSection, 1));
367   PetscCall(PetscSectionSetChart(coordSection, 0, numVertices));
368   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
369   for (v = 0; v < numVertices; ++v) {
370     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
371     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
372   }
373   PetscCall(PetscSectionSetUp(coordSection));
374   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
375   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
376   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
377   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
378   PetscCall(VecSetBlockSize(coordinates, cdim));
379   PetscCall(VecSetType(coordinates, VECSTANDARD));
380   PetscCall(VecGetArray(coordinates, &coords));
381   coords[0] = lower[0];
382   coords[1] = upper[0];
383   PetscCall(VecRestoreArray(coordinates, &coords));
384   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
385   PetscCall(VecDestroy(&coordinates));
386   PetscFunctionReturn(0);
387 }
388 
389 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
390 {
391   const PetscInt numVertices    = (edges[0] + 1) * (edges[1] + 1);
392   const PetscInt numEdges       = edges[0] * (edges[1] + 1) + (edges[0] + 1) * edges[1];
393   PetscInt       markerTop      = 1;
394   PetscInt       markerBottom   = 1;
395   PetscInt       markerRight    = 1;
396   PetscInt       markerLeft     = 1;
397   PetscBool      markerSeparate = PETSC_FALSE;
398   Vec            coordinates;
399   PetscSection   coordSection;
400   PetscScalar   *coords;
401   PetscInt       coordSize;
402   PetscMPIInt    rank;
403   PetscInt       v, vx, vy;
404 
405   PetscFunctionBegin;
406   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
407   if (markerSeparate) {
408     markerTop    = 3;
409     markerBottom = 1;
410     markerRight  = 2;
411     markerLeft   = 4;
412   }
413   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
414   if (rank == 0) {
415     PetscInt e, ex, ey;
416 
417     PetscCall(DMPlexSetChart(dm, 0, numEdges + numVertices));
418     for (e = 0; e < numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
419     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
420     for (vx = 0; vx <= edges[0]; vx++) {
421       for (ey = 0; ey < edges[1]; ey++) {
422         PetscInt edge   = vx * edges[1] + ey + edges[0] * (edges[1] + 1);
423         PetscInt vertex = ey * (edges[0] + 1) + vx + numEdges;
424         PetscInt cone[2];
425 
426         cone[0] = vertex;
427         cone[1] = vertex + edges[0] + 1;
428         PetscCall(DMPlexSetCone(dm, edge, cone));
429         if (vx == edges[0]) {
430           PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
431           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
432           if (ey == edges[1] - 1) {
433             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
434             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerRight));
435           }
436         } else if (vx == 0) {
437           PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
438           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
439           if (ey == edges[1] - 1) {
440             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
441             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft));
442           }
443         }
444       }
445     }
446     for (vy = 0; vy <= edges[1]; vy++) {
447       for (ex = 0; ex < edges[0]; ex++) {
448         PetscInt edge   = vy * edges[0] + ex;
449         PetscInt vertex = vy * (edges[0] + 1) + ex + numEdges;
450         PetscInt cone[2];
451 
452         cone[0] = vertex;
453         cone[1] = vertex + 1;
454         PetscCall(DMPlexSetCone(dm, edge, cone));
455         if (vy == edges[1]) {
456           PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
457           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
458           if (ex == edges[0] - 1) {
459             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
460             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerTop));
461           }
462         } else if (vy == 0) {
463           PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
464           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
465           if (ex == edges[0] - 1) {
466             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
467             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom));
468           }
469         }
470       }
471     }
472   }
473   PetscCall(DMPlexSymmetrize(dm));
474   PetscCall(DMPlexStratify(dm));
475   /* Build coordinates */
476   PetscCall(DMSetCoordinateDim(dm, 2));
477   PetscCall(DMGetCoordinateSection(dm, &coordSection));
478   PetscCall(PetscSectionSetNumFields(coordSection, 1));
479   PetscCall(PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices));
480   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 2));
481   for (v = numEdges; v < numEdges + numVertices; ++v) {
482     PetscCall(PetscSectionSetDof(coordSection, v, 2));
483     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 2));
484   }
485   PetscCall(PetscSectionSetUp(coordSection));
486   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
487   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
488   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
489   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
490   PetscCall(VecSetBlockSize(coordinates, 2));
491   PetscCall(VecSetType(coordinates, VECSTANDARD));
492   PetscCall(VecGetArray(coordinates, &coords));
493   for (vy = 0; vy <= edges[1]; ++vy) {
494     for (vx = 0; vx <= edges[0]; ++vx) {
495       coords[(vy * (edges[0] + 1) + vx) * 2 + 0] = lower[0] + ((upper[0] - lower[0]) / edges[0]) * vx;
496       coords[(vy * (edges[0] + 1) + vx) * 2 + 1] = lower[1] + ((upper[1] - lower[1]) / edges[1]) * vy;
497     }
498   }
499   PetscCall(VecRestoreArray(coordinates, &coords));
500   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
501   PetscCall(VecDestroy(&coordinates));
502   PetscFunctionReturn(0);
503 }
504 
505 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
506 {
507   PetscInt     vertices[3], numVertices;
508   PetscInt     numFaces       = 2 * faces[0] * faces[1] + 2 * faces[1] * faces[2] + 2 * faces[0] * faces[2];
509   PetscInt     markerTop      = 1;
510   PetscInt     markerBottom   = 1;
511   PetscInt     markerFront    = 1;
512   PetscInt     markerBack     = 1;
513   PetscInt     markerRight    = 1;
514   PetscInt     markerLeft     = 1;
515   PetscBool    markerSeparate = PETSC_FALSE;
516   Vec          coordinates;
517   PetscSection coordSection;
518   PetscScalar *coords;
519   PetscInt     coordSize;
520   PetscMPIInt  rank;
521   PetscInt     v, vx, vy, vz;
522   PetscInt     voffset, iface = 0, cone[4];
523 
524   PetscFunctionBegin;
525   PetscCheck(faces[0] >= 1 && faces[1] >= 1 && faces[2] >= 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
526   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
527   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
528   if (markerSeparate) {
529     markerBottom = 1;
530     markerTop    = 2;
531     markerFront  = 3;
532     markerBack   = 4;
533     markerRight  = 5;
534     markerLeft   = 6;
535   }
536   vertices[0] = faces[0] + 1;
537   vertices[1] = faces[1] + 1;
538   vertices[2] = faces[2] + 1;
539   numVertices = vertices[0] * vertices[1] * vertices[2];
540   if (rank == 0) {
541     PetscInt f;
542 
543     PetscCall(DMPlexSetChart(dm, 0, numFaces + numVertices));
544     for (f = 0; f < numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
545     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
546 
547     /* Side 0 (Top) */
548     for (vy = 0; vy < faces[1]; vy++) {
549       for (vx = 0; vx < faces[0]; vx++) {
550         voffset = numFaces + vertices[0] * vertices[1] * (vertices[2] - 1) + vy * vertices[0] + vx;
551         cone[0] = voffset;
552         cone[1] = voffset + 1;
553         cone[2] = voffset + vertices[0] + 1;
554         cone[3] = voffset + vertices[0];
555         PetscCall(DMPlexSetCone(dm, iface, cone));
556         PetscCall(DMSetLabelValue(dm, "marker", iface, markerTop));
557         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerTop));
558         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerTop));
559         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerTop));
560         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerTop));
561         iface++;
562       }
563     }
564 
565     /* Side 1 (Bottom) */
566     for (vy = 0; vy < faces[1]; vy++) {
567       for (vx = 0; vx < faces[0]; vx++) {
568         voffset = numFaces + vy * (faces[0] + 1) + vx;
569         cone[0] = voffset + 1;
570         cone[1] = voffset;
571         cone[2] = voffset + vertices[0];
572         cone[3] = voffset + vertices[0] + 1;
573         PetscCall(DMPlexSetCone(dm, iface, cone));
574         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBottom));
575         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBottom));
576         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBottom));
577         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerBottom));
578         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerBottom));
579         iface++;
580       }
581     }
582 
583     /* Side 2 (Front) */
584     for (vz = 0; vz < faces[2]; vz++) {
585       for (vx = 0; vx < faces[0]; vx++) {
586         voffset = numFaces + vz * vertices[0] * vertices[1] + vx;
587         cone[0] = voffset;
588         cone[1] = voffset + 1;
589         cone[2] = voffset + vertices[0] * vertices[1] + 1;
590         cone[3] = voffset + vertices[0] * vertices[1];
591         PetscCall(DMPlexSetCone(dm, iface, cone));
592         PetscCall(DMSetLabelValue(dm, "marker", iface, markerFront));
593         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerFront));
594         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerFront));
595         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerFront));
596         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerFront));
597         iface++;
598       }
599     }
600 
601     /* Side 3 (Back) */
602     for (vz = 0; vz < faces[2]; vz++) {
603       for (vx = 0; vx < faces[0]; vx++) {
604         voffset = numFaces + vz * vertices[0] * vertices[1] + vertices[0] * (vertices[1] - 1) + vx;
605         cone[0] = voffset + vertices[0] * vertices[1];
606         cone[1] = voffset + vertices[0] * vertices[1] + 1;
607         cone[2] = voffset + 1;
608         cone[3] = voffset;
609         PetscCall(DMPlexSetCone(dm, iface, cone));
610         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBack));
611         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBack));
612         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBack));
613         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerBack));
614         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerBack));
615         iface++;
616       }
617     }
618 
619     /* Side 4 (Left) */
620     for (vz = 0; vz < faces[2]; vz++) {
621       for (vy = 0; vy < faces[1]; vy++) {
622         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0];
623         cone[0] = voffset;
624         cone[1] = voffset + vertices[0] * vertices[1];
625         cone[2] = voffset + vertices[0] * vertices[1] + vertices[0];
626         cone[3] = voffset + vertices[0];
627         PetscCall(DMPlexSetCone(dm, iface, cone));
628         PetscCall(DMSetLabelValue(dm, "marker", iface, markerLeft));
629         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerLeft));
630         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerLeft));
631         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[1] + 0, markerLeft));
632         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerLeft));
633         iface++;
634       }
635     }
636 
637     /* Side 5 (Right) */
638     for (vz = 0; vz < faces[2]; vz++) {
639       for (vy = 0; vy < faces[1]; vy++) {
640         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0] + faces[0];
641         cone[0] = voffset + vertices[0] * vertices[1];
642         cone[1] = voffset;
643         cone[2] = voffset + vertices[0];
644         cone[3] = voffset + vertices[0] * vertices[1] + vertices[0];
645         PetscCall(DMPlexSetCone(dm, iface, cone));
646         PetscCall(DMSetLabelValue(dm, "marker", iface, markerRight));
647         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerRight));
648         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerRight));
649         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerRight));
650         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerRight));
651         iface++;
652       }
653     }
654   }
655   PetscCall(DMPlexSymmetrize(dm));
656   PetscCall(DMPlexStratify(dm));
657   /* Build coordinates */
658   PetscCall(DMSetCoordinateDim(dm, 3));
659   PetscCall(DMGetCoordinateSection(dm, &coordSection));
660   PetscCall(PetscSectionSetNumFields(coordSection, 1));
661   PetscCall(PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices));
662   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 3));
663   for (v = numFaces; v < numFaces + numVertices; ++v) {
664     PetscCall(PetscSectionSetDof(coordSection, v, 3));
665     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 3));
666   }
667   PetscCall(PetscSectionSetUp(coordSection));
668   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
669   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
670   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
671   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
672   PetscCall(VecSetBlockSize(coordinates, 3));
673   PetscCall(VecSetType(coordinates, VECSTANDARD));
674   PetscCall(VecGetArray(coordinates, &coords));
675   for (vz = 0; vz <= faces[2]; ++vz) {
676     for (vy = 0; vy <= faces[1]; ++vy) {
677       for (vx = 0; vx <= faces[0]; ++vx) {
678         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 0] = lower[0] + ((upper[0] - lower[0]) / faces[0]) * vx;
679         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 1] = lower[1] + ((upper[1] - lower[1]) / faces[1]) * vy;
680         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 2] = lower[2] + ((upper[2] - lower[2]) / faces[2]) * vz;
681       }
682     }
683   }
684   PetscCall(VecRestoreArray(coordinates, &coords));
685   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
686   PetscCall(VecDestroy(&coordinates));
687   PetscFunctionReturn(0);
688 }
689 
690 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate)
691 {
692   PetscFunctionBegin;
693   PetscValidLogicalCollectiveInt(dm, dim, 2);
694   PetscCall(DMSetDimension(dm, dim - 1));
695   PetscCall(DMSetCoordinateDim(dm, dim));
696   switch (dim) {
697   case 1:
698     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(dm, lower, upper, faces));
699     break;
700   case 2:
701     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(dm, lower, upper, faces));
702     break;
703   case 3:
704     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(dm, lower, upper, faces));
705     break;
706   default:
707     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Dimension not supported: %" PetscInt_FMT, dim);
708   }
709   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
710   PetscFunctionReturn(0);
711 }
712 
713 /*@C
714   DMPlexCreateBoxSurfaceMesh - Creates a mesh on the surface of the tensor product of unit intervals (box) using tensor cells (hexahedra).
715 
716   Collective
717 
718   Input Parameters:
719 + comm        - The communicator for the DM object
720 . dim         - The spatial dimension of the box, so the resulting mesh is has dimension dim-1
721 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
722 . lower       - The lower left corner, or NULL for (0, 0, 0)
723 . upper       - The upper right corner, or NULL for (1, 1, 1)
724 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
725 
726   Output Parameter:
727 . dm  - The DM object
728 
729   Level: beginner
730 
731 .seealso: `DMSetFromOptions()`, `DMPlexCreateBoxMesh()`, `DMPlexCreateFromFile()`, `DMSetType()`, `DMCreate()`
732 @*/
733 PetscErrorCode DMPlexCreateBoxSurfaceMesh(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate, DM *dm)
734 {
735   PetscInt  fac[3] = {1, 1, 1};
736   PetscReal low[3] = {0, 0, 0};
737   PetscReal upp[3] = {1, 1, 1};
738 
739   PetscFunctionBegin;
740   PetscCall(DMCreate(comm, dm));
741   PetscCall(DMSetType(*dm, DMPLEX));
742   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(*dm, dim, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, interpolate));
743   PetscFunctionReturn(0);
744 }
745 
746 static PetscErrorCode DMPlexCreateLineMesh_Internal(DM dm, PetscInt segments, PetscReal lower, PetscReal upper, DMBoundaryType bd)
747 {
748   PetscInt     i, fStart, fEnd, numCells = 0, numVerts = 0;
749   PetscInt     numPoints[2], *coneSize, *cones, *coneOrientations;
750   PetscScalar *vertexCoords;
751   PetscReal    L, maxCell;
752   PetscBool    markerSeparate = PETSC_FALSE;
753   PetscInt     markerLeft = 1, faceMarkerLeft = 1;
754   PetscInt     markerRight = 1, faceMarkerRight = 2;
755   PetscBool    wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
756   PetscMPIInt  rank;
757 
758   PetscFunctionBegin;
759   PetscValidPointer(dm, 1);
760 
761   PetscCall(DMSetDimension(dm, 1));
762   PetscCall(DMCreateLabel(dm, "marker"));
763   PetscCall(DMCreateLabel(dm, "Face Sets"));
764 
765   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
766   if (rank == 0) numCells = segments;
767   if (rank == 0) numVerts = segments + (wrap ? 0 : 1);
768 
769   numPoints[0] = numVerts;
770   numPoints[1] = numCells;
771   PetscCall(PetscMalloc4(numCells + numVerts, &coneSize, numCells * 2, &cones, numCells + numVerts, &coneOrientations, numVerts, &vertexCoords));
772   PetscCall(PetscArrayzero(coneOrientations, numCells + numVerts));
773   for (i = 0; i < numCells; ++i) coneSize[i] = 2;
774   for (i = 0; i < numVerts; ++i) coneSize[numCells + i] = 0;
775   for (i = 0; i < numCells; ++i) {
776     cones[2 * i]     = numCells + i % numVerts;
777     cones[2 * i + 1] = numCells + (i + 1) % numVerts;
778   }
779   for (i = 0; i < numVerts; ++i) vertexCoords[i] = lower + (upper - lower) * ((PetscReal)i / (PetscReal)numCells);
780   PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
781   PetscCall(PetscFree4(coneSize, cones, coneOrientations, vertexCoords));
782 
783   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
784   if (markerSeparate) {
785     markerLeft  = faceMarkerLeft;
786     markerRight = faceMarkerRight;
787   }
788   if (!wrap && rank == 0) {
789     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
790     PetscCall(DMSetLabelValue(dm, "marker", fStart, markerLeft));
791     PetscCall(DMSetLabelValue(dm, "marker", fEnd - 1, markerRight));
792     PetscCall(DMSetLabelValue(dm, "Face Sets", fStart, faceMarkerLeft));
793     PetscCall(DMSetLabelValue(dm, "Face Sets", fEnd - 1, faceMarkerRight));
794   }
795   if (wrap) {
796     L       = upper - lower;
797     maxCell = (PetscReal)1.1 * (L / (PetscReal)PetscMax(1, segments));
798     PetscCall(DMSetPeriodicity(dm, &maxCell, &lower, &L));
799   }
800   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
801   PetscFunctionReturn(0);
802 }
803 
804 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
805 {
806   DM      boundary, vol;
807   DMLabel bdlabel;
808 
809   PetscFunctionBegin;
810   PetscValidPointer(dm, 1);
811   for (PetscInt i = 0; i < dim; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
812   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &boundary));
813   PetscCall(DMSetType(boundary, DMPLEX));
814   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(boundary, dim, faces, lower, upper, PETSC_FALSE));
815   PetscCall(DMPlexGenerate(boundary, NULL, interpolate, &vol));
816   PetscCall(DMGetLabel(vol, "marker", &bdlabel));
817   if (bdlabel) PetscCall(DMPlexLabelComplete(vol, bdlabel));
818   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_FALSE, vol));
819   PetscCall(DMPlexReplace_Internal(dm, &vol));
820   PetscCall(DMDestroy(&boundary));
821   PetscFunctionReturn(0);
822 }
823 
824 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
825 {
826   DMLabel     cutLabel  = NULL;
827   PetscInt    markerTop = 1, faceMarkerTop = 1;
828   PetscInt    markerBottom = 1, faceMarkerBottom = 1;
829   PetscInt    markerFront = 1, faceMarkerFront = 1;
830   PetscInt    markerBack = 1, faceMarkerBack = 1;
831   PetscInt    markerRight = 1, faceMarkerRight = 1;
832   PetscInt    markerLeft = 1, faceMarkerLeft = 1;
833   PetscInt    dim;
834   PetscBool   markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
835   PetscMPIInt rank;
836 
837   PetscFunctionBegin;
838   PetscCall(DMGetDimension(dm, &dim));
839   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
840   PetscCall(DMCreateLabel(dm, "marker"));
841   PetscCall(DMCreateLabel(dm, "Face Sets"));
842   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
843   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST || bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST || bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
844     if (cutMarker) {
845       PetscCall(DMCreateLabel(dm, "periodic_cut"));
846       PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
847     }
848   }
849   switch (dim) {
850   case 2:
851     faceMarkerTop    = 3;
852     faceMarkerBottom = 1;
853     faceMarkerRight  = 2;
854     faceMarkerLeft   = 4;
855     break;
856   case 3:
857     faceMarkerBottom = 1;
858     faceMarkerTop    = 2;
859     faceMarkerFront  = 3;
860     faceMarkerBack   = 4;
861     faceMarkerRight  = 5;
862     faceMarkerLeft   = 6;
863     break;
864   default:
865     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
866   }
867   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
868   if (markerSeparate) {
869     markerBottom = faceMarkerBottom;
870     markerTop    = faceMarkerTop;
871     markerFront  = faceMarkerFront;
872     markerBack   = faceMarkerBack;
873     markerRight  = faceMarkerRight;
874     markerLeft   = faceMarkerLeft;
875   }
876   {
877     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
878     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
879     const PetscInt numZEdges    = rank == 0 ? edges[2] : 0;
880     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0] + 1) : 0;
881     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1] + 1) : 0;
882     const PetscInt numZVertices = rank == 0 ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2] + 1) : 0;
883     const PetscInt numCells     = numXEdges * numYEdges * numZEdges;
884     const PetscInt numXFaces    = numYEdges * numZEdges;
885     const PetscInt numYFaces    = numXEdges * numZEdges;
886     const PetscInt numZFaces    = numXEdges * numYEdges;
887     const PetscInt numTotXFaces = numXVertices * numXFaces;
888     const PetscInt numTotYFaces = numYVertices * numYFaces;
889     const PetscInt numTotZFaces = numZVertices * numZFaces;
890     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
891     const PetscInt numTotXEdges = numXEdges * numYVertices * numZVertices;
892     const PetscInt numTotYEdges = numYEdges * numXVertices * numZVertices;
893     const PetscInt numTotZEdges = numZEdges * numXVertices * numYVertices;
894     const PetscInt numVertices  = numXVertices * numYVertices * numZVertices;
895     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
896     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
897     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
898     const PetscInt firstYFace   = firstXFace + numTotXFaces;
899     const PetscInt firstZFace   = firstYFace + numTotYFaces;
900     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
901     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
902     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
903     Vec            coordinates;
904     PetscSection   coordSection;
905     PetscScalar   *coords;
906     PetscInt       coordSize;
907     PetscInt       v, vx, vy, vz;
908     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
909 
910     PetscCall(DMPlexSetChart(dm, 0, numCells + numFaces + numEdges + numVertices));
911     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
912     for (f = firstXFace; f < firstXFace + numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
913     for (e = firstXEdge; e < firstXEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
914     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
915     /* Build cells */
916     for (fz = 0; fz < numZEdges; ++fz) {
917       for (fy = 0; fy < numYEdges; ++fy) {
918         for (fx = 0; fx < numXEdges; ++fx) {
919           PetscInt cell  = (fz * numYEdges + fy) * numXEdges + fx;
920           PetscInt faceB = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
921           PetscInt faceT = firstZFace + (fy * numXEdges + fx) * numZVertices + ((fz + 1) % numZVertices);
922           PetscInt faceF = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
923           PetscInt faceK = firstYFace + (fz * numXEdges + fx) * numYVertices + ((fy + 1) % numYVertices);
924           PetscInt faceL = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
925           PetscInt faceR = firstXFace + (fz * numYEdges + fy) * numXVertices + ((fx + 1) % numXVertices);
926           /* B,  T,  F,  K,  R,  L */
927           PetscInt ornt[6] = {-2, 0, 0, -3, 0, -2}; /* ??? */
928           PetscInt cone[6];
929 
930           /* no boundary twisting in 3D */
931           cone[0] = faceB;
932           cone[1] = faceT;
933           cone[2] = faceF;
934           cone[3] = faceK;
935           cone[4] = faceR;
936           cone[5] = faceL;
937           PetscCall(DMPlexSetCone(dm, cell, cone));
938           PetscCall(DMPlexSetConeOrientation(dm, cell, ornt));
939           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
940           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
941           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
942         }
943       }
944     }
945     /* Build x faces */
946     for (fz = 0; fz < numZEdges; ++fz) {
947       for (fy = 0; fy < numYEdges; ++fy) {
948         for (fx = 0; fx < numXVertices; ++fx) {
949           PetscInt face    = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
950           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
951           PetscInt edgeR   = firstZEdge + (((fy + 1) % numYVertices) * numXVertices + fx) * numZEdges + fz;
952           PetscInt edgeB   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
953           PetscInt edgeT   = firstYEdge + (((fz + 1) % numZVertices) * numXVertices + fx) * numYEdges + fy;
954           PetscInt ornt[4] = {0, 0, -1, -1};
955           PetscInt cone[4];
956 
957           if (dim == 3) {
958             /* markers */
959             if (bdX != DM_BOUNDARY_PERIODIC) {
960               if (fx == numXVertices - 1) {
961                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight));
962                 PetscCall(DMSetLabelValue(dm, "marker", face, markerRight));
963               } else if (fx == 0) {
964                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft));
965                 PetscCall(DMSetLabelValue(dm, "marker", face, markerLeft));
966               }
967             }
968           }
969           cone[0] = edgeB;
970           cone[1] = edgeR;
971           cone[2] = edgeT;
972           cone[3] = edgeL;
973           PetscCall(DMPlexSetCone(dm, face, cone));
974           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
975         }
976       }
977     }
978     /* Build y faces */
979     for (fz = 0; fz < numZEdges; ++fz) {
980       for (fx = 0; fx < numXEdges; ++fx) {
981         for (fy = 0; fy < numYVertices; ++fy) {
982           PetscInt face    = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
983           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
984           PetscInt edgeR   = firstZEdge + (fy * numXVertices + ((fx + 1) % numXVertices)) * numZEdges + fz;
985           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
986           PetscInt edgeT   = firstXEdge + (((fz + 1) % numZVertices) * numYVertices + fy) * numXEdges + fx;
987           PetscInt ornt[4] = {0, 0, -1, -1};
988           PetscInt cone[4];
989 
990           if (dim == 3) {
991             /* markers */
992             if (bdY != DM_BOUNDARY_PERIODIC) {
993               if (fy == numYVertices - 1) {
994                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack));
995                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBack));
996               } else if (fy == 0) {
997                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront));
998                 PetscCall(DMSetLabelValue(dm, "marker", face, markerFront));
999               }
1000             }
1001           }
1002           cone[0] = edgeB;
1003           cone[1] = edgeR;
1004           cone[2] = edgeT;
1005           cone[3] = edgeL;
1006           PetscCall(DMPlexSetCone(dm, face, cone));
1007           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1008         }
1009       }
1010     }
1011     /* Build z faces */
1012     for (fy = 0; fy < numYEdges; ++fy) {
1013       for (fx = 0; fx < numXEdges; ++fx) {
1014         for (fz = 0; fz < numZVertices; fz++) {
1015           PetscInt face    = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
1016           PetscInt edgeL   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
1017           PetscInt edgeR   = firstYEdge + (fz * numXVertices + ((fx + 1) % numXVertices)) * numYEdges + fy;
1018           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1019           PetscInt edgeT   = firstXEdge + (fz * numYVertices + ((fy + 1) % numYVertices)) * numXEdges + fx;
1020           PetscInt ornt[4] = {0, 0, -1, -1};
1021           PetscInt cone[4];
1022 
1023           if (dim == 2) {
1024             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges - 1) {
1025               edgeR += numYEdges - 1 - 2 * fy;
1026               ornt[1] = -1;
1027             }
1028             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges - 1) {
1029               edgeT += numXEdges - 1 - 2 * fx;
1030               ornt[2] = 0;
1031             }
1032             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1033             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1034           } else {
1035             /* markers */
1036             if (bdZ != DM_BOUNDARY_PERIODIC) {
1037               if (fz == numZVertices - 1) {
1038                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop));
1039                 PetscCall(DMSetLabelValue(dm, "marker", face, markerTop));
1040               } else if (fz == 0) {
1041                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom));
1042                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBottom));
1043               }
1044             }
1045           }
1046           cone[0] = edgeB;
1047           cone[1] = edgeR;
1048           cone[2] = edgeT;
1049           cone[3] = edgeL;
1050           PetscCall(DMPlexSetCone(dm, face, cone));
1051           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1052         }
1053       }
1054     }
1055     /* Build Z edges*/
1056     for (vy = 0; vy < numYVertices; vy++) {
1057       for (vx = 0; vx < numXVertices; vx++) {
1058         for (ez = 0; ez < numZEdges; ez++) {
1059           const PetscInt edge    = firstZEdge + (vy * numXVertices + vx) * numZEdges + ez;
1060           const PetscInt vertexB = firstVertex + (ez * numYVertices + vy) * numXVertices + vx;
1061           const PetscInt vertexT = firstVertex + (((ez + 1) % numZVertices) * numYVertices + vy) * numXVertices + vx;
1062           PetscInt       cone[2];
1063 
1064           cone[0] = vertexB;
1065           cone[1] = vertexT;
1066           PetscCall(DMPlexSetCone(dm, edge, cone));
1067           if (dim == 3) {
1068             if (bdX != DM_BOUNDARY_PERIODIC) {
1069               if (vx == numXVertices - 1) {
1070                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1071                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1072                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1073               } else if (vx == 0) {
1074                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1075                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1076                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1077               }
1078             }
1079             if (bdY != DM_BOUNDARY_PERIODIC) {
1080               if (vy == numYVertices - 1) {
1081                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1082                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1083                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1084               } else if (vy == 0) {
1085                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1086                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1087                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1088               }
1089             }
1090           }
1091         }
1092       }
1093     }
1094     /* Build Y edges*/
1095     for (vz = 0; vz < numZVertices; vz++) {
1096       for (vx = 0; vx < numXVertices; vx++) {
1097         for (ey = 0; ey < numYEdges; ey++) {
1098           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges - 1) ? (numXVertices - vx - 1) : (vz * numYVertices + ((ey + 1) % numYVertices)) * numXVertices + vx;
1099           const PetscInt edge    = firstYEdge + (vz * numXVertices + vx) * numYEdges + ey;
1100           const PetscInt vertexF = firstVertex + (vz * numYVertices + ey) * numXVertices + vx;
1101           const PetscInt vertexK = firstVertex + nextv;
1102           PetscInt       cone[2];
1103 
1104           cone[0] = vertexF;
1105           cone[1] = vertexK;
1106           PetscCall(DMPlexSetCone(dm, edge, cone));
1107           if (dim == 2) {
1108             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1109               if (vx == numXVertices - 1) {
1110                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight));
1111                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1112                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1113                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1114               } else if (vx == 0) {
1115                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft));
1116                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1117                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1118                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1119               }
1120             } else {
1121               if (vx == 0 && cutLabel) {
1122                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1123                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1124                 if (ey == numYEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1125               }
1126             }
1127           } else {
1128             if (bdX != DM_BOUNDARY_PERIODIC) {
1129               if (vx == numXVertices - 1) {
1130                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1131                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1132                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1133               } else if (vx == 0) {
1134                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1135                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1136                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1137               }
1138             }
1139             if (bdZ != DM_BOUNDARY_PERIODIC) {
1140               if (vz == numZVertices - 1) {
1141                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1142                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1143                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1144               } else if (vz == 0) {
1145                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1146                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1147                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1148               }
1149             }
1150           }
1151         }
1152       }
1153     }
1154     /* Build X edges*/
1155     for (vz = 0; vz < numZVertices; vz++) {
1156       for (vy = 0; vy < numYVertices; vy++) {
1157         for (ex = 0; ex < numXEdges; ex++) {
1158           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges - 1) ? (numYVertices - vy - 1) * numXVertices : (vz * numYVertices + vy) * numXVertices + (ex + 1) % numXVertices;
1159           const PetscInt edge    = firstXEdge + (vz * numYVertices + vy) * numXEdges + ex;
1160           const PetscInt vertexL = firstVertex + (vz * numYVertices + vy) * numXVertices + ex;
1161           const PetscInt vertexR = firstVertex + nextv;
1162           PetscInt       cone[2];
1163 
1164           cone[0] = vertexL;
1165           cone[1] = vertexR;
1166           PetscCall(DMPlexSetCone(dm, edge, cone));
1167           if (dim == 2) {
1168             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1169               if (vy == numYVertices - 1) {
1170                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop));
1171                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1172                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1173                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1174               } else if (vy == 0) {
1175                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom));
1176                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1177                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1178                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1179               }
1180             } else {
1181               if (vy == 0 && cutLabel) {
1182                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1183                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1184                 if (ex == numXEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1185               }
1186             }
1187           } else {
1188             if (bdY != DM_BOUNDARY_PERIODIC) {
1189               if (vy == numYVertices - 1) {
1190                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1191                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1192                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1193               } else if (vy == 0) {
1194                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1195                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1196                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1197               }
1198             }
1199             if (bdZ != DM_BOUNDARY_PERIODIC) {
1200               if (vz == numZVertices - 1) {
1201                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1202                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1203                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1204               } else if (vz == 0) {
1205                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1206                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1207                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1208               }
1209             }
1210           }
1211         }
1212       }
1213     }
1214     PetscCall(DMPlexSymmetrize(dm));
1215     PetscCall(DMPlexStratify(dm));
1216     /* Build coordinates */
1217     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1218     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1219     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1220     PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVertices));
1221     for (v = firstVertex; v < firstVertex + numVertices; ++v) {
1222       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1223       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1224     }
1225     PetscCall(PetscSectionSetUp(coordSection));
1226     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1227     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1228     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1229     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1230     PetscCall(VecSetBlockSize(coordinates, dim));
1231     PetscCall(VecSetType(coordinates, VECSTANDARD));
1232     PetscCall(VecGetArray(coordinates, &coords));
1233     for (vz = 0; vz < numZVertices; ++vz) {
1234       for (vy = 0; vy < numYVertices; ++vy) {
1235         for (vx = 0; vx < numXVertices; ++vx) {
1236           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * vx;
1237           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * vy;
1238           if (dim == 3) coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 2] = lower[2] + ((upper[2] - lower[2]) / numZEdges) * vz;
1239         }
1240       }
1241     }
1242     PetscCall(VecRestoreArray(coordinates, &coords));
1243     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1244     PetscCall(VecDestroy(&coordinates));
1245   }
1246   PetscFunctionReturn(0);
1247 }
1248 
1249 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1250 {
1251   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1252   PetscInt       fac[3] = {0, 0, 0}, d;
1253 
1254   PetscFunctionBegin;
1255   PetscValidPointer(dm, 1);
1256   PetscValidLogicalCollectiveInt(dm, dim, 2);
1257   PetscCall(DMSetDimension(dm, dim));
1258   for (d = 0; d < dim; ++d) {
1259     fac[d] = faces[d];
1260     bdt[d] = periodicity[d];
1261   }
1262   PetscCall(DMPlexCreateCubeMesh_Internal(dm, lower, upper, fac, bdt[0], bdt[1], bdt[2]));
1263   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST || periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST || (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
1264     PetscReal L[3]       = {-1., -1., 0.};
1265     PetscReal maxCell[3] = {-1., -1., 0.};
1266 
1267     for (d = 0; d < dim; ++d) {
1268       if (periodicity[d] != DM_BOUNDARY_NONE) {
1269         L[d]       = upper[d] - lower[d];
1270         maxCell[d] = 1.1 * (L[d] / PetscMax(1, faces[d]));
1271       }
1272     }
1273     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1274   }
1275   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
1276   PetscFunctionReturn(0);
1277 }
1278 
1279 static PetscErrorCode DMPlexCreateBoxMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1280 {
1281   PetscFunctionBegin;
1282   if (dim == 1) PetscCall(DMPlexCreateLineMesh_Internal(dm, faces[0], lower[0], upper[0], periodicity[0]));
1283   else if (simplex) PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1284   else PetscCall(DMPlexCreateBoxMesh_Tensor_Internal(dm, dim, faces, lower, upper, periodicity));
1285   if (!interpolate && dim > 1 && !simplex) {
1286     DM udm;
1287 
1288     PetscCall(DMPlexUninterpolate(dm, &udm));
1289     PetscCall(DMPlexCopyCoordinates(dm, udm));
1290     PetscCall(DMPlexReplace_Internal(dm, &udm));
1291   }
1292   PetscFunctionReturn(0);
1293 }
1294 
1295 /*@C
1296   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
1297 
1298   Collective
1299 
1300   Input Parameters:
1301 + comm        - The communicator for the DM object
1302 . dim         - The spatial dimension
1303 . simplex     - PETSC_TRUE for simplices, PETSC_FALSE for tensor cells
1304 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1305 . lower       - The lower left corner, or NULL for (0, 0, 0)
1306 . upper       - The upper right corner, or NULL for (1, 1, 1)
1307 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
1308 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1309 
1310   Output Parameter:
1311 . dm  - The DM object
1312 
1313   Note: If you want to customize this mesh using options, you just need to
1314 $  DMCreate(comm, &dm);
1315 $  DMSetType(dm, DMPLEX);
1316 $  DMSetFromOptions(dm);
1317 and use the options on the DMSetFromOptions() page.
1318 
1319   Here is the numbering returned for 2 faces in each direction for tensor cells:
1320 $ 10---17---11---18----12
1321 $  |         |         |
1322 $  |         |         |
1323 $ 20    2   22    3    24
1324 $  |         |         |
1325 $  |         |         |
1326 $  7---15----8---16----9
1327 $  |         |         |
1328 $  |         |         |
1329 $ 19    0   21    1   23
1330 $  |         |         |
1331 $  |         |         |
1332 $  4---13----5---14----6
1333 
1334 and for simplicial cells
1335 
1336 $ 14----8---15----9----16
1337 $  |\     5  |\      7 |
1338 $  | \       | \       |
1339 $ 13   2    14    3    15
1340 $  | 4   \   | 6   \   |
1341 $  |       \ |       \ |
1342 $ 11----6---12----7----13
1343 $  |\        |\        |
1344 $  | \    1  | \     3 |
1345 $ 10   0    11    1    12
1346 $  | 0   \   | 2   \   |
1347 $  |       \ |       \ |
1348 $  8----4----9----5----10
1349 
1350   Level: beginner
1351 
1352 .seealso: `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
1353 @*/
1354 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
1355 {
1356   PetscInt       fac[3] = {1, 1, 1};
1357   PetscReal      low[3] = {0, 0, 0};
1358   PetscReal      upp[3] = {1, 1, 1};
1359   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1360 
1361   PetscFunctionBegin;
1362   PetscCall(DMCreate(comm, dm));
1363   PetscCall(DMSetType(*dm, DMPLEX));
1364   PetscCall(DMPlexCreateBoxMesh_Internal(*dm, dim, simplex, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt, interpolate));
1365   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1366   PetscFunctionReturn(0);
1367 }
1368 
1369 static PetscErrorCode DMPlexCreateWedgeBoxMesh_Internal(DM dm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1370 {
1371   DM       bdm, vol;
1372   PetscInt i;
1373 
1374   PetscFunctionBegin;
1375   for (i = 0; i < 3; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity not yet supported");
1376   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &bdm));
1377   PetscCall(DMSetType(bdm, DMPLEX));
1378   PetscCall(DMSetDimension(bdm, 2));
1379   PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(bdm, 2, faces, lower, upper, periodicity, PETSC_TRUE));
1380   PetscCall(DMPlexExtrude(bdm, faces[2], upper[2] - lower[2], PETSC_TRUE, PETSC_FALSE, NULL, NULL, &vol));
1381   PetscCall(DMDestroy(&bdm));
1382   PetscCall(DMPlexReplace_Internal(dm, &vol));
1383   if (lower[2] != 0.0) {
1384     Vec          v;
1385     PetscScalar *x;
1386     PetscInt     cDim, n;
1387 
1388     PetscCall(DMGetCoordinatesLocal(dm, &v));
1389     PetscCall(VecGetBlockSize(v, &cDim));
1390     PetscCall(VecGetLocalSize(v, &n));
1391     PetscCall(VecGetArray(v, &x));
1392     x += cDim;
1393     for (i = 0; i < n; i += cDim) x[i] += lower[2];
1394     PetscCall(VecRestoreArray(v, &x));
1395     PetscCall(DMSetCoordinatesLocal(dm, v));
1396   }
1397   PetscFunctionReturn(0);
1398 }
1399 
1400 /*@
1401   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tesselating the (x,y) plane and extruding in the third direction using wedge cells.
1402 
1403   Collective
1404 
1405   Input Parameters:
1406 + comm        - The communicator for the DM object
1407 . faces       - Number of faces per dimension, or NULL for (1, 1, 1)
1408 . lower       - The lower left corner, or NULL for (0, 0, 0)
1409 . upper       - The upper right corner, or NULL for (1, 1, 1)
1410 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
1411 . orderHeight - If PETSC_TRUE, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1412 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1413 
1414   Output Parameter:
1415 . dm  - The DM object
1416 
1417   Level: beginner
1418 
1419 .seealso: `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateWedgeCylinderMesh()`, `DMExtrude()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1420 @*/
1421 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool orderHeight, PetscBool interpolate, DM *dm)
1422 {
1423   PetscInt       fac[3] = {1, 1, 1};
1424   PetscReal      low[3] = {0, 0, 0};
1425   PetscReal      upp[3] = {1, 1, 1};
1426   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1427 
1428   PetscFunctionBegin;
1429   PetscCall(DMCreate(comm, dm));
1430   PetscCall(DMSetType(*dm, DMPLEX));
1431   PetscCall(DMPlexCreateWedgeBoxMesh_Internal(*dm, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt));
1432   if (!interpolate) {
1433     DM udm;
1434 
1435     PetscCall(DMPlexUninterpolate(*dm, &udm));
1436     PetscCall(DMPlexReplace_Internal(*dm, &udm));
1437   }
1438   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1439   PetscFunctionReturn(0);
1440 }
1441 
1442 /*@C
1443   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all DM options in the database.
1444 
1445   Logically Collective on dm
1446 
1447   Input Parameters:
1448 + dm - the DM context
1449 - prefix - the prefix to prepend to all option names
1450 
1451   Notes:
1452   A hyphen (-) must NOT be given at the beginning of the prefix name.
1453   The first character of all runtime options is AUTOMATICALLY the hyphen.
1454 
1455   Level: advanced
1456 
1457 .seealso: `SNESSetFromOptions()`
1458 @*/
1459 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
1460 {
1461   DM_Plex *mesh = (DM_Plex *)dm->data;
1462 
1463   PetscFunctionBegin;
1464   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1465   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, prefix));
1466   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)mesh->partitioner, prefix));
1467   PetscFunctionReturn(0);
1468 }
1469 
1470 /* Remap geometry to cylinder
1471    TODO: This only works for a single refinement, then it is broken
1472 
1473      Interior square: Linear interpolation is correct
1474      The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
1475      such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
1476 
1477        phi     = arctan(y/x)
1478        d_close = sqrt(1/8 + 1/4 sin^2(phi))
1479        d_far   = sqrt(1/2 + sin^2(phi))
1480 
1481      so we remap them using
1482 
1483        x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
1484        y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
1485 
1486      If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
1487 */
1488 static void snapToCylinder(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
1489 {
1490   const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1491   const PetscReal ds2 = 0.5 * dis;
1492 
1493   if ((PetscAbsScalar(u[0]) <= ds2) && (PetscAbsScalar(u[1]) <= ds2)) {
1494     f0[0] = u[0];
1495     f0[1] = u[1];
1496   } else {
1497     PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
1498 
1499     x    = PetscRealPart(u[0]);
1500     y    = PetscRealPart(u[1]);
1501     phi  = PetscAtan2Real(y, x);
1502     sinp = PetscSinReal(phi);
1503     cosp = PetscCosReal(phi);
1504     if ((PetscAbsReal(phi) > PETSC_PI / 4.0) && (PetscAbsReal(phi) < 3.0 * PETSC_PI / 4.0)) {
1505       dc = PetscAbsReal(ds2 / sinp);
1506       df = PetscAbsReal(dis / sinp);
1507       xc = ds2 * x / PetscAbsReal(y);
1508       yc = ds2 * PetscSignReal(y);
1509     } else {
1510       dc = PetscAbsReal(ds2 / cosp);
1511       df = PetscAbsReal(dis / cosp);
1512       xc = ds2 * PetscSignReal(x);
1513       yc = ds2 * y / PetscAbsReal(x);
1514     }
1515     f0[0] = xc + (u[0] - xc) * (1.0 - dc) / (df - dc);
1516     f0[1] = yc + (u[1] - yc) * (1.0 - dc) / (df - dc);
1517   }
1518   f0[2] = u[2];
1519 }
1520 
1521 static PetscErrorCode DMPlexCreateHexCylinderMesh_Internal(DM dm, DMBoundaryType periodicZ)
1522 {
1523   const PetscInt dim = 3;
1524   PetscInt       numCells, numVertices;
1525   PetscMPIInt    rank;
1526 
1527   PetscFunctionBegin;
1528   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1529   PetscCall(DMSetDimension(dm, dim));
1530   /* Create topology */
1531   {
1532     PetscInt cone[8], c;
1533 
1534     numCells    = rank == 0 ? 5 : 0;
1535     numVertices = rank == 0 ? 16 : 0;
1536     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1537       numCells *= 3;
1538       numVertices = rank == 0 ? 24 : 0;
1539     }
1540     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1541     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 8));
1542     PetscCall(DMSetUp(dm));
1543     if (rank == 0) {
1544       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1545         cone[0] = 15;
1546         cone[1] = 18;
1547         cone[2] = 17;
1548         cone[3] = 16;
1549         cone[4] = 31;
1550         cone[5] = 32;
1551         cone[6] = 33;
1552         cone[7] = 34;
1553         PetscCall(DMPlexSetCone(dm, 0, cone));
1554         cone[0] = 16;
1555         cone[1] = 17;
1556         cone[2] = 24;
1557         cone[3] = 23;
1558         cone[4] = 32;
1559         cone[5] = 36;
1560         cone[6] = 37;
1561         cone[7] = 33; /* 22 25 26 21 */
1562         PetscCall(DMPlexSetCone(dm, 1, cone));
1563         cone[0] = 18;
1564         cone[1] = 27;
1565         cone[2] = 24;
1566         cone[3] = 17;
1567         cone[4] = 34;
1568         cone[5] = 33;
1569         cone[6] = 37;
1570         cone[7] = 38;
1571         PetscCall(DMPlexSetCone(dm, 2, cone));
1572         cone[0] = 29;
1573         cone[1] = 27;
1574         cone[2] = 18;
1575         cone[3] = 15;
1576         cone[4] = 35;
1577         cone[5] = 31;
1578         cone[6] = 34;
1579         cone[7] = 38;
1580         PetscCall(DMPlexSetCone(dm, 3, cone));
1581         cone[0] = 29;
1582         cone[1] = 15;
1583         cone[2] = 16;
1584         cone[3] = 23;
1585         cone[4] = 35;
1586         cone[5] = 36;
1587         cone[6] = 32;
1588         cone[7] = 31;
1589         PetscCall(DMPlexSetCone(dm, 4, cone));
1590 
1591         cone[0] = 31;
1592         cone[1] = 34;
1593         cone[2] = 33;
1594         cone[3] = 32;
1595         cone[4] = 19;
1596         cone[5] = 22;
1597         cone[6] = 21;
1598         cone[7] = 20;
1599         PetscCall(DMPlexSetCone(dm, 5, cone));
1600         cone[0] = 32;
1601         cone[1] = 33;
1602         cone[2] = 37;
1603         cone[3] = 36;
1604         cone[4] = 22;
1605         cone[5] = 25;
1606         cone[6] = 26;
1607         cone[7] = 21;
1608         PetscCall(DMPlexSetCone(dm, 6, cone));
1609         cone[0] = 34;
1610         cone[1] = 38;
1611         cone[2] = 37;
1612         cone[3] = 33;
1613         cone[4] = 20;
1614         cone[5] = 21;
1615         cone[6] = 26;
1616         cone[7] = 28;
1617         PetscCall(DMPlexSetCone(dm, 7, cone));
1618         cone[0] = 35;
1619         cone[1] = 38;
1620         cone[2] = 34;
1621         cone[3] = 31;
1622         cone[4] = 30;
1623         cone[5] = 19;
1624         cone[6] = 20;
1625         cone[7] = 28;
1626         PetscCall(DMPlexSetCone(dm, 8, cone));
1627         cone[0] = 35;
1628         cone[1] = 31;
1629         cone[2] = 32;
1630         cone[3] = 36;
1631         cone[4] = 30;
1632         cone[5] = 25;
1633         cone[6] = 22;
1634         cone[7] = 19;
1635         PetscCall(DMPlexSetCone(dm, 9, cone));
1636 
1637         cone[0] = 19;
1638         cone[1] = 20;
1639         cone[2] = 21;
1640         cone[3] = 22;
1641         cone[4] = 15;
1642         cone[5] = 16;
1643         cone[6] = 17;
1644         cone[7] = 18;
1645         PetscCall(DMPlexSetCone(dm, 10, cone));
1646         cone[0] = 22;
1647         cone[1] = 21;
1648         cone[2] = 26;
1649         cone[3] = 25;
1650         cone[4] = 16;
1651         cone[5] = 23;
1652         cone[6] = 24;
1653         cone[7] = 17;
1654         PetscCall(DMPlexSetCone(dm, 11, cone));
1655         cone[0] = 20;
1656         cone[1] = 28;
1657         cone[2] = 26;
1658         cone[3] = 21;
1659         cone[4] = 18;
1660         cone[5] = 17;
1661         cone[6] = 24;
1662         cone[7] = 27;
1663         PetscCall(DMPlexSetCone(dm, 12, cone));
1664         cone[0] = 30;
1665         cone[1] = 28;
1666         cone[2] = 20;
1667         cone[3] = 19;
1668         cone[4] = 29;
1669         cone[5] = 15;
1670         cone[6] = 18;
1671         cone[7] = 27;
1672         PetscCall(DMPlexSetCone(dm, 13, cone));
1673         cone[0] = 30;
1674         cone[1] = 19;
1675         cone[2] = 22;
1676         cone[3] = 25;
1677         cone[4] = 29;
1678         cone[5] = 23;
1679         cone[6] = 16;
1680         cone[7] = 15;
1681         PetscCall(DMPlexSetCone(dm, 14, cone));
1682       } else {
1683         cone[0] = 5;
1684         cone[1] = 8;
1685         cone[2] = 7;
1686         cone[3] = 6;
1687         cone[4] = 9;
1688         cone[5] = 12;
1689         cone[6] = 11;
1690         cone[7] = 10;
1691         PetscCall(DMPlexSetCone(dm, 0, cone));
1692         cone[0] = 6;
1693         cone[1] = 7;
1694         cone[2] = 14;
1695         cone[3] = 13;
1696         cone[4] = 12;
1697         cone[5] = 15;
1698         cone[6] = 16;
1699         cone[7] = 11;
1700         PetscCall(DMPlexSetCone(dm, 1, cone));
1701         cone[0] = 8;
1702         cone[1] = 17;
1703         cone[2] = 14;
1704         cone[3] = 7;
1705         cone[4] = 10;
1706         cone[5] = 11;
1707         cone[6] = 16;
1708         cone[7] = 18;
1709         PetscCall(DMPlexSetCone(dm, 2, cone));
1710         cone[0] = 19;
1711         cone[1] = 17;
1712         cone[2] = 8;
1713         cone[3] = 5;
1714         cone[4] = 20;
1715         cone[5] = 9;
1716         cone[6] = 10;
1717         cone[7] = 18;
1718         PetscCall(DMPlexSetCone(dm, 3, cone));
1719         cone[0] = 19;
1720         cone[1] = 5;
1721         cone[2] = 6;
1722         cone[3] = 13;
1723         cone[4] = 20;
1724         cone[5] = 15;
1725         cone[6] = 12;
1726         cone[7] = 9;
1727         PetscCall(DMPlexSetCone(dm, 4, cone));
1728       }
1729     }
1730     PetscCall(DMPlexSymmetrize(dm));
1731     PetscCall(DMPlexStratify(dm));
1732   }
1733   /* Create cube geometry */
1734   {
1735     Vec             coordinates;
1736     PetscSection    coordSection;
1737     PetscScalar    *coords;
1738     PetscInt        coordSize, v;
1739     const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1740     const PetscReal ds2 = dis / 2.0;
1741 
1742     /* Build coordinates */
1743     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1744     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1745     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1746     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1747     for (v = numCells; v < numCells + numVertices; ++v) {
1748       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1749       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1750     }
1751     PetscCall(PetscSectionSetUp(coordSection));
1752     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1753     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1754     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1755     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1756     PetscCall(VecSetBlockSize(coordinates, dim));
1757     PetscCall(VecSetType(coordinates, VECSTANDARD));
1758     PetscCall(VecGetArray(coordinates, &coords));
1759     if (rank == 0) {
1760       coords[0 * dim + 0]  = -ds2;
1761       coords[0 * dim + 1]  = -ds2;
1762       coords[0 * dim + 2]  = 0.0;
1763       coords[1 * dim + 0]  = ds2;
1764       coords[1 * dim + 1]  = -ds2;
1765       coords[1 * dim + 2]  = 0.0;
1766       coords[2 * dim + 0]  = ds2;
1767       coords[2 * dim + 1]  = ds2;
1768       coords[2 * dim + 2]  = 0.0;
1769       coords[3 * dim + 0]  = -ds2;
1770       coords[3 * dim + 1]  = ds2;
1771       coords[3 * dim + 2]  = 0.0;
1772       coords[4 * dim + 0]  = -ds2;
1773       coords[4 * dim + 1]  = -ds2;
1774       coords[4 * dim + 2]  = 1.0;
1775       coords[5 * dim + 0]  = -ds2;
1776       coords[5 * dim + 1]  = ds2;
1777       coords[5 * dim + 2]  = 1.0;
1778       coords[6 * dim + 0]  = ds2;
1779       coords[6 * dim + 1]  = ds2;
1780       coords[6 * dim + 2]  = 1.0;
1781       coords[7 * dim + 0]  = ds2;
1782       coords[7 * dim + 1]  = -ds2;
1783       coords[7 * dim + 2]  = 1.0;
1784       coords[8 * dim + 0]  = dis;
1785       coords[8 * dim + 1]  = -dis;
1786       coords[8 * dim + 2]  = 0.0;
1787       coords[9 * dim + 0]  = dis;
1788       coords[9 * dim + 1]  = dis;
1789       coords[9 * dim + 2]  = 0.0;
1790       coords[10 * dim + 0] = dis;
1791       coords[10 * dim + 1] = -dis;
1792       coords[10 * dim + 2] = 1.0;
1793       coords[11 * dim + 0] = dis;
1794       coords[11 * dim + 1] = dis;
1795       coords[11 * dim + 2] = 1.0;
1796       coords[12 * dim + 0] = -dis;
1797       coords[12 * dim + 1] = dis;
1798       coords[12 * dim + 2] = 0.0;
1799       coords[13 * dim + 0] = -dis;
1800       coords[13 * dim + 1] = dis;
1801       coords[13 * dim + 2] = 1.0;
1802       coords[14 * dim + 0] = -dis;
1803       coords[14 * dim + 1] = -dis;
1804       coords[14 * dim + 2] = 0.0;
1805       coords[15 * dim + 0] = -dis;
1806       coords[15 * dim + 1] = -dis;
1807       coords[15 * dim + 2] = 1.0;
1808       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1809         /* 15 31 19 */ coords[16 * dim + 0] = -ds2;
1810         coords[16 * dim + 1]                = -ds2;
1811         coords[16 * dim + 2]                = 0.5;
1812         /* 16 32 22 */ coords[17 * dim + 0] = ds2;
1813         coords[17 * dim + 1]                = -ds2;
1814         coords[17 * dim + 2]                = 0.5;
1815         /* 17 33 21 */ coords[18 * dim + 0] = ds2;
1816         coords[18 * dim + 1]                = ds2;
1817         coords[18 * dim + 2]                = 0.5;
1818         /* 18 34 20 */ coords[19 * dim + 0] = -ds2;
1819         coords[19 * dim + 1]                = ds2;
1820         coords[19 * dim + 2]                = 0.5;
1821         /* 29 35 30 */ coords[20 * dim + 0] = -dis;
1822         coords[20 * dim + 1]                = -dis;
1823         coords[20 * dim + 2]                = 0.5;
1824         /* 23 36 25 */ coords[21 * dim + 0] = dis;
1825         coords[21 * dim + 1]                = -dis;
1826         coords[21 * dim + 2]                = 0.5;
1827         /* 24 37 26 */ coords[22 * dim + 0] = dis;
1828         coords[22 * dim + 1]                = dis;
1829         coords[22 * dim + 2]                = 0.5;
1830         /* 27 38 28 */ coords[23 * dim + 0] = -dis;
1831         coords[23 * dim + 1]                = dis;
1832         coords[23 * dim + 2]                = 0.5;
1833       }
1834     }
1835     PetscCall(VecRestoreArray(coordinates, &coords));
1836     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1837     PetscCall(VecDestroy(&coordinates));
1838   }
1839   /* Create periodicity */
1840   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1841     PetscReal L[3]       = {-1., -1., 0.};
1842     PetscReal maxCell[3] = {-1., -1., 0.};
1843     PetscReal lower[3]   = {0.0, 0.0, 0.0};
1844     PetscReal upper[3]   = {1.0, 1.0, 1.5};
1845     PetscInt  numZCells  = 3;
1846 
1847     L[2]       = upper[2] - lower[2];
1848     maxCell[2] = 1.1 * (L[2] / numZCells);
1849     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1850   }
1851   {
1852     DM          cdm;
1853     PetscDS     cds;
1854     PetscScalar c[2] = {1.0, 1.0};
1855 
1856     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToCylinder));
1857     PetscCall(DMGetCoordinateDM(dm, &cdm));
1858     PetscCall(DMGetDS(cdm, &cds));
1859     PetscCall(PetscDSSetConstants(cds, 2, c));
1860   }
1861   /* Wait for coordinate creation before doing in-place modification */
1862   PetscCall(DMPlexInterpolateInPlace_Internal(dm));
1863   PetscFunctionReturn(0);
1864 }
1865 
1866 /*@
1867   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
1868 
1869   Collective
1870 
1871   Input Parameters:
1872 + comm      - The communicator for the DM object
1873 - periodicZ - The boundary type for the Z direction
1874 
1875   Output Parameter:
1876 . dm  - The DM object
1877 
1878   Note:
1879   Here is the output numbering looking from the bottom of the cylinder:
1880 $       17-----14
1881 $        |     |
1882 $        |  2  |
1883 $        |     |
1884 $ 17-----8-----7-----14
1885 $  |     |     |     |
1886 $  |  3  |  0  |  1  |
1887 $  |     |     |     |
1888 $ 19-----5-----6-----13
1889 $        |     |
1890 $        |  4  |
1891 $        |     |
1892 $       19-----13
1893 $
1894 $ and up through the top
1895 $
1896 $       18-----16
1897 $        |     |
1898 $        |  2  |
1899 $        |     |
1900 $ 18----10----11-----16
1901 $  |     |     |     |
1902 $  |  3  |  0  |  1  |
1903 $  |     |     |     |
1904 $ 20-----9----12-----15
1905 $        |     |
1906 $        |  4  |
1907 $        |     |
1908 $       20-----15
1909 
1910   Level: beginner
1911 
1912 .seealso: `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1913 @*/
1914 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, DMBoundaryType periodicZ, DM *dm)
1915 {
1916   PetscFunctionBegin;
1917   PetscValidPointer(dm, 3);
1918   PetscCall(DMCreate(comm, dm));
1919   PetscCall(DMSetType(*dm, DMPLEX));
1920   PetscCall(DMPlexCreateHexCylinderMesh_Internal(*dm, periodicZ));
1921   PetscFunctionReturn(0);
1922 }
1923 
1924 static PetscErrorCode DMPlexCreateWedgeCylinderMesh_Internal(DM dm, PetscInt n, PetscBool interpolate)
1925 {
1926   const PetscInt dim = 3;
1927   PetscInt       numCells, numVertices, v;
1928   PetscMPIInt    rank;
1929 
1930   PetscFunctionBegin;
1931   PetscCheck(n >= 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %" PetscInt_FMT " cannot be negative", n);
1932   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1933   PetscCall(DMSetDimension(dm, dim));
1934   /* Must create the celltype label here so that we do not automatically try to compute the types */
1935   PetscCall(DMCreateLabel(dm, "celltype"));
1936   /* Create topology */
1937   {
1938     PetscInt cone[6], c;
1939 
1940     numCells    = rank == 0 ? n : 0;
1941     numVertices = rank == 0 ? 2 * (n + 1) : 0;
1942     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1943     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
1944     PetscCall(DMSetUp(dm));
1945     for (c = 0; c < numCells; c++) {
1946       cone[0] = c + n * 1;
1947       cone[1] = (c + 1) % n + n * 1;
1948       cone[2] = 0 + 3 * n;
1949       cone[3] = c + n * 2;
1950       cone[4] = (c + 1) % n + n * 2;
1951       cone[5] = 1 + 3 * n;
1952       PetscCall(DMPlexSetCone(dm, c, cone));
1953       PetscCall(DMPlexSetCellType(dm, c, DM_POLYTOPE_TRI_PRISM_TENSOR));
1954     }
1955     PetscCall(DMPlexSymmetrize(dm));
1956     PetscCall(DMPlexStratify(dm));
1957   }
1958   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetCellType(dm, v, DM_POLYTOPE_POINT));
1959   /* Create cylinder geometry */
1960   {
1961     Vec          coordinates;
1962     PetscSection coordSection;
1963     PetscScalar *coords;
1964     PetscInt     coordSize, c;
1965 
1966     /* Build coordinates */
1967     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1968     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1969     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1970     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1971     for (v = numCells; v < numCells + numVertices; ++v) {
1972       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1973       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1974     }
1975     PetscCall(PetscSectionSetUp(coordSection));
1976     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1977     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1978     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1979     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1980     PetscCall(VecSetBlockSize(coordinates, dim));
1981     PetscCall(VecSetType(coordinates, VECSTANDARD));
1982     PetscCall(VecGetArray(coordinates, &coords));
1983     for (c = 0; c < numCells; c++) {
1984       coords[(c + 0 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
1985       coords[(c + 0 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
1986       coords[(c + 0 * n) * dim + 2] = 1.0;
1987       coords[(c + 1 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
1988       coords[(c + 1 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
1989       coords[(c + 1 * n) * dim + 2] = 0.0;
1990     }
1991     if (rank == 0) {
1992       coords[(2 * n + 0) * dim + 0] = 0.0;
1993       coords[(2 * n + 0) * dim + 1] = 0.0;
1994       coords[(2 * n + 0) * dim + 2] = 1.0;
1995       coords[(2 * n + 1) * dim + 0] = 0.0;
1996       coords[(2 * n + 1) * dim + 1] = 0.0;
1997       coords[(2 * n + 1) * dim + 2] = 0.0;
1998     }
1999     PetscCall(VecRestoreArray(coordinates, &coords));
2000     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2001     PetscCall(VecDestroy(&coordinates));
2002   }
2003   /* Interpolate */
2004   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2005   PetscFunctionReturn(0);
2006 }
2007 
2008 /*@
2009   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
2010 
2011   Collective
2012 
2013   Input Parameters:
2014 + comm - The communicator for the DM object
2015 . n    - The number of wedges around the origin
2016 - interpolate - Create edges and faces
2017 
2018   Output Parameter:
2019 . dm  - The DM object
2020 
2021   Level: beginner
2022 
2023 .seealso: `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2024 @*/
2025 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
2026 {
2027   PetscFunctionBegin;
2028   PetscValidPointer(dm, 4);
2029   PetscCall(DMCreate(comm, dm));
2030   PetscCall(DMSetType(*dm, DMPLEX));
2031   PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(*dm, n, interpolate));
2032   PetscFunctionReturn(0);
2033 }
2034 
2035 static inline PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2036 {
2037   PetscReal prod = 0.0;
2038   PetscInt  i;
2039   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
2040   return PetscSqrtReal(prod);
2041 }
2042 static inline PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2043 {
2044   PetscReal prod = 0.0;
2045   PetscInt  i;
2046   for (i = 0; i < dim; ++i) prod += x[i] * y[i];
2047   return prod;
2048 }
2049 
2050 /* The first constant is the sphere radius */
2051 static void snapToSphere(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
2052 {
2053   PetscReal r     = PetscRealPart(constants[0]);
2054   PetscReal norm2 = 0.0, fac;
2055   PetscInt  n     = uOff[1] - uOff[0], d;
2056 
2057   for (d = 0; d < n; ++d) norm2 += PetscSqr(PetscRealPart(u[d]));
2058   fac = r / PetscSqrtReal(norm2);
2059   for (d = 0; d < n; ++d) f0[d] = u[d] * fac;
2060 }
2061 
2062 static PetscErrorCode DMPlexCreateSphereMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, PetscReal R)
2063 {
2064   const PetscInt embedDim = dim + 1;
2065   PetscSection   coordSection;
2066   Vec            coordinates;
2067   PetscScalar   *coords;
2068   PetscReal     *coordsIn;
2069   PetscInt       numCells, numEdges, numVerts = 0, firstVertex = 0, v, firstEdge, coordSize, d, c, e;
2070   PetscMPIInt    rank;
2071 
2072   PetscFunctionBegin;
2073   PetscValidLogicalCollectiveBool(dm, simplex, 3);
2074   PetscCall(DMSetDimension(dm, dim));
2075   PetscCall(DMSetCoordinateDim(dm, dim + 1));
2076   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2077   switch (dim) {
2078   case 2:
2079     if (simplex) {
2080       const PetscReal radius    = PetscSqrtReal(1 + PETSC_PHI * PETSC_PHI) / (1.0 + PETSC_PHI);
2081       const PetscReal edgeLen   = 2.0 / (1.0 + PETSC_PHI) * (R / radius);
2082       const PetscInt  degree    = 5;
2083       PetscReal       vertex[3] = {0.0, 1.0 / (1.0 + PETSC_PHI), PETSC_PHI / (1.0 + PETSC_PHI)};
2084       PetscInt        s[3]      = {1, 1, 1};
2085       PetscInt        cone[3];
2086       PetscInt       *graph, p, i, j, k;
2087 
2088       vertex[0] *= R / radius;
2089       vertex[1] *= R / radius;
2090       vertex[2] *= R / radius;
2091       numCells    = rank == 0 ? 20 : 0;
2092       numVerts    = rank == 0 ? 12 : 0;
2093       firstVertex = numCells;
2094       /* Use icosahedron, which for a R-sphere has coordinates which are all cyclic permutations of
2095 
2096            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
2097 
2098          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2099          length is then given by 2/(1+\phi) = 2 * 0.38197 = 0.76393.
2100       */
2101       /* Construct vertices */
2102       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2103       if (rank == 0) {
2104         for (p = 0, i = 0; p < embedDim; ++p) {
2105           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2106             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2107               for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertex[(d + p) % embedDim];
2108               ++i;
2109             }
2110           }
2111         }
2112       }
2113       /* Construct graph */
2114       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2115       for (i = 0; i < numVerts; ++i) {
2116         for (j = 0, k = 0; j < numVerts; ++j) {
2117           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2118             graph[i * numVerts + j] = 1;
2119             ++k;
2120           }
2121         }
2122         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid icosahedron, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2123       }
2124       /* Build Topology */
2125       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2126       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2127       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2128       /* Cells */
2129       for (i = 0, c = 0; i < numVerts; ++i) {
2130         for (j = 0; j < i; ++j) {
2131           for (k = 0; k < j; ++k) {
2132             if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i]) {
2133               cone[0] = firstVertex + i;
2134               cone[1] = firstVertex + j;
2135               cone[2] = firstVertex + k;
2136               /* Check orientation */
2137               {
2138                 const PetscInt epsilon[3][3][3] = {
2139                   {{0, 0, 0},  {0, 0, 1},  {0, -1, 0}},
2140                   {{0, 0, -1}, {0, 0, 0},  {1, 0, 0} },
2141                   {{0, 1, 0},  {-1, 0, 0}, {0, 0, 0} }
2142                 };
2143                 PetscReal normal[3];
2144                 PetscInt  e, f;
2145 
2146                 for (d = 0; d < embedDim; ++d) {
2147                   normal[d] = 0.0;
2148                   for (e = 0; e < embedDim; ++e) {
2149                     for (f = 0; f < embedDim; ++f) normal[d] += epsilon[d][e][f] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]);
2150                   }
2151                 }
2152                 if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2153                   PetscInt tmp = cone[1];
2154                   cone[1]      = cone[2];
2155                   cone[2]      = tmp;
2156                 }
2157               }
2158               PetscCall(DMPlexSetCone(dm, c++, cone));
2159             }
2160           }
2161         }
2162       }
2163       PetscCall(DMPlexSymmetrize(dm));
2164       PetscCall(DMPlexStratify(dm));
2165       PetscCall(PetscFree(graph));
2166     } else {
2167       /*
2168         12-21--13
2169          |     |
2170         25  4  24
2171          |     |
2172   12-25--9-16--8-24--13
2173    |     |     |     |
2174   23  5 17  0 15  3  22
2175    |     |     |     |
2176   10-20--6-14--7-19--11
2177          |     |
2178         20  1  19
2179          |     |
2180         10-18--11
2181          |     |
2182         23  2  22
2183          |     |
2184         12-21--13
2185        */
2186       PetscInt cone[4], ornt[4];
2187 
2188       numCells    = rank == 0 ? 6 : 0;
2189       numEdges    = rank == 0 ? 12 : 0;
2190       numVerts    = rank == 0 ? 8 : 0;
2191       firstVertex = numCells;
2192       firstEdge   = numCells + numVerts;
2193       /* Build Topology */
2194       PetscCall(DMPlexSetChart(dm, 0, numCells + numEdges + numVerts));
2195       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 4));
2196       for (e = firstEdge; e < firstEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
2197       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2198       if (rank == 0) {
2199         /* Cell 0 */
2200         cone[0] = 14;
2201         cone[1] = 15;
2202         cone[2] = 16;
2203         cone[3] = 17;
2204         PetscCall(DMPlexSetCone(dm, 0, cone));
2205         ornt[0] = 0;
2206         ornt[1] = 0;
2207         ornt[2] = 0;
2208         ornt[3] = 0;
2209         PetscCall(DMPlexSetConeOrientation(dm, 0, ornt));
2210         /* Cell 1 */
2211         cone[0] = 18;
2212         cone[1] = 19;
2213         cone[2] = 14;
2214         cone[3] = 20;
2215         PetscCall(DMPlexSetCone(dm, 1, cone));
2216         ornt[0] = 0;
2217         ornt[1] = 0;
2218         ornt[2] = -1;
2219         ornt[3] = 0;
2220         PetscCall(DMPlexSetConeOrientation(dm, 1, ornt));
2221         /* Cell 2 */
2222         cone[0] = 21;
2223         cone[1] = 22;
2224         cone[2] = 18;
2225         cone[3] = 23;
2226         PetscCall(DMPlexSetCone(dm, 2, cone));
2227         ornt[0] = 0;
2228         ornt[1] = 0;
2229         ornt[2] = -1;
2230         ornt[3] = 0;
2231         PetscCall(DMPlexSetConeOrientation(dm, 2, ornt));
2232         /* Cell 3 */
2233         cone[0] = 19;
2234         cone[1] = 22;
2235         cone[2] = 24;
2236         cone[3] = 15;
2237         PetscCall(DMPlexSetCone(dm, 3, cone));
2238         ornt[0] = -1;
2239         ornt[1] = -1;
2240         ornt[2] = 0;
2241         ornt[3] = -1;
2242         PetscCall(DMPlexSetConeOrientation(dm, 3, ornt));
2243         /* Cell 4 */
2244         cone[0] = 16;
2245         cone[1] = 24;
2246         cone[2] = 21;
2247         cone[3] = 25;
2248         PetscCall(DMPlexSetCone(dm, 4, cone));
2249         ornt[0] = -1;
2250         ornt[1] = -1;
2251         ornt[2] = -1;
2252         ornt[3] = 0;
2253         PetscCall(DMPlexSetConeOrientation(dm, 4, ornt));
2254         /* Cell 5 */
2255         cone[0] = 20;
2256         cone[1] = 17;
2257         cone[2] = 25;
2258         cone[3] = 23;
2259         PetscCall(DMPlexSetCone(dm, 5, cone));
2260         ornt[0] = -1;
2261         ornt[1] = -1;
2262         ornt[2] = -1;
2263         ornt[3] = -1;
2264         PetscCall(DMPlexSetConeOrientation(dm, 5, ornt));
2265         /* Edges */
2266         cone[0] = 6;
2267         cone[1] = 7;
2268         PetscCall(DMPlexSetCone(dm, 14, cone));
2269         cone[0] = 7;
2270         cone[1] = 8;
2271         PetscCall(DMPlexSetCone(dm, 15, cone));
2272         cone[0] = 8;
2273         cone[1] = 9;
2274         PetscCall(DMPlexSetCone(dm, 16, cone));
2275         cone[0] = 9;
2276         cone[1] = 6;
2277         PetscCall(DMPlexSetCone(dm, 17, cone));
2278         cone[0] = 10;
2279         cone[1] = 11;
2280         PetscCall(DMPlexSetCone(dm, 18, cone));
2281         cone[0] = 11;
2282         cone[1] = 7;
2283         PetscCall(DMPlexSetCone(dm, 19, cone));
2284         cone[0] = 6;
2285         cone[1] = 10;
2286         PetscCall(DMPlexSetCone(dm, 20, cone));
2287         cone[0] = 12;
2288         cone[1] = 13;
2289         PetscCall(DMPlexSetCone(dm, 21, cone));
2290         cone[0] = 13;
2291         cone[1] = 11;
2292         PetscCall(DMPlexSetCone(dm, 22, cone));
2293         cone[0] = 10;
2294         cone[1] = 12;
2295         PetscCall(DMPlexSetCone(dm, 23, cone));
2296         cone[0] = 13;
2297         cone[1] = 8;
2298         PetscCall(DMPlexSetCone(dm, 24, cone));
2299         cone[0] = 12;
2300         cone[1] = 9;
2301         PetscCall(DMPlexSetCone(dm, 25, cone));
2302       }
2303       PetscCall(DMPlexSymmetrize(dm));
2304       PetscCall(DMPlexStratify(dm));
2305       /* Build coordinates */
2306       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2307       if (rank == 0) {
2308         coordsIn[0 * embedDim + 0] = -R;
2309         coordsIn[0 * embedDim + 1] = R;
2310         coordsIn[0 * embedDim + 2] = -R;
2311         coordsIn[1 * embedDim + 0] = R;
2312         coordsIn[1 * embedDim + 1] = R;
2313         coordsIn[1 * embedDim + 2] = -R;
2314         coordsIn[2 * embedDim + 0] = R;
2315         coordsIn[2 * embedDim + 1] = -R;
2316         coordsIn[2 * embedDim + 2] = -R;
2317         coordsIn[3 * embedDim + 0] = -R;
2318         coordsIn[3 * embedDim + 1] = -R;
2319         coordsIn[3 * embedDim + 2] = -R;
2320         coordsIn[4 * embedDim + 0] = -R;
2321         coordsIn[4 * embedDim + 1] = R;
2322         coordsIn[4 * embedDim + 2] = R;
2323         coordsIn[5 * embedDim + 0] = R;
2324         coordsIn[5 * embedDim + 1] = R;
2325         coordsIn[5 * embedDim + 2] = R;
2326         coordsIn[6 * embedDim + 0] = -R;
2327         coordsIn[6 * embedDim + 1] = -R;
2328         coordsIn[6 * embedDim + 2] = R;
2329         coordsIn[7 * embedDim + 0] = R;
2330         coordsIn[7 * embedDim + 1] = -R;
2331         coordsIn[7 * embedDim + 2] = R;
2332       }
2333     }
2334     break;
2335   case 3:
2336     if (simplex) {
2337       const PetscReal edgeLen         = 1.0 / PETSC_PHI;
2338       PetscReal       vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
2339       PetscReal       vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
2340       PetscReal       vertexC[4]      = {0.5, 0.5 * PETSC_PHI, 0.5 / PETSC_PHI, 0.0};
2341       const PetscInt  degree          = 12;
2342       PetscInt        s[4]            = {1, 1, 1};
2343       PetscInt        evenPerm[12][4] = {
2344         {0, 1, 2, 3},
2345         {0, 2, 3, 1},
2346         {0, 3, 1, 2},
2347         {1, 0, 3, 2},
2348         {1, 2, 0, 3},
2349         {1, 3, 2, 0},
2350         {2, 0, 1, 3},
2351         {2, 1, 3, 0},
2352         {2, 3, 0, 1},
2353         {3, 0, 2, 1},
2354         {3, 1, 0, 2},
2355         {3, 2, 1, 0}
2356       };
2357       PetscInt  cone[4];
2358       PetscInt *graph, p, i, j, k, l;
2359 
2360       vertexA[0] *= R;
2361       vertexA[1] *= R;
2362       vertexA[2] *= R;
2363       vertexA[3] *= R;
2364       vertexB[0] *= R;
2365       vertexB[1] *= R;
2366       vertexB[2] *= R;
2367       vertexB[3] *= R;
2368       vertexC[0] *= R;
2369       vertexC[1] *= R;
2370       vertexC[2] *= R;
2371       vertexC[3] *= R;
2372       numCells    = rank == 0 ? 600 : 0;
2373       numVerts    = rank == 0 ? 120 : 0;
2374       firstVertex = numCells;
2375       /* Use the 600-cell, which for a unit sphere has coordinates which are
2376 
2377            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
2378                (\pm 1,    0,       0,      0)  all cyclic permutations   8
2379            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
2380 
2381          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2382          length is then given by 1/\phi = 0.61803.
2383 
2384          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
2385          http://mathworld.wolfram.com/600-Cell.html
2386       */
2387       /* Construct vertices */
2388       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2389       i = 0;
2390       if (rank == 0) {
2391         for (s[0] = -1; s[0] < 2; s[0] += 2) {
2392           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2393             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2394               for (s[3] = -1; s[3] < 2; s[3] += 2) {
2395                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[d] * vertexA[d];
2396                 ++i;
2397               }
2398             }
2399           }
2400         }
2401         for (p = 0; p < embedDim; ++p) {
2402           s[1] = s[2] = s[3] = 1;
2403           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2404             for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertexB[(d + p) % embedDim];
2405             ++i;
2406           }
2407         }
2408         for (p = 0; p < 12; ++p) {
2409           s[3] = 1;
2410           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2411             for (s[1] = -1; s[1] < 2; s[1] += 2) {
2412               for (s[2] = -1; s[2] < 2; s[2] += 2) {
2413                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[evenPerm[p][d]] * vertexC[evenPerm[p][d]];
2414                 ++i;
2415               }
2416             }
2417           }
2418         }
2419       }
2420       PetscCheck(i == numVerts, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertices %" PetscInt_FMT " != %" PetscInt_FMT, i, numVerts);
2421       /* Construct graph */
2422       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2423       for (i = 0; i < numVerts; ++i) {
2424         for (j = 0, k = 0; j < numVerts; ++j) {
2425           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2426             graph[i * numVerts + j] = 1;
2427             ++k;
2428           }
2429         }
2430         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2431       }
2432       /* Build Topology */
2433       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2434       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2435       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2436       /* Cells */
2437       if (rank == 0) {
2438         for (i = 0, c = 0; i < numVerts; ++i) {
2439           for (j = 0; j < i; ++j) {
2440             for (k = 0; k < j; ++k) {
2441               for (l = 0; l < k; ++l) {
2442                 if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i] && graph[l * numVerts + i] && graph[l * numVerts + j] && graph[l * numVerts + k]) {
2443                   cone[0] = firstVertex + i;
2444                   cone[1] = firstVertex + j;
2445                   cone[2] = firstVertex + k;
2446                   cone[3] = firstVertex + l;
2447                   /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
2448                   {
2449                     const PetscInt epsilon[4][4][4][4] = {
2450                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, -1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 0, 0}, {0, 1, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 1, 0}, {0, -1, 0, 0}, {0, 0, 0, 0}}},
2451 
2452                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}}, {{0, 0, -1, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}}},
2453 
2454                       {{{0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, 0, 0}, {0, -1, 0, 0}}, {{0, 0, 0, -1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 1, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}},
2455 
2456                       {{{0, 0, 0, 0}, {0, 0, -1, 0}, {0, 1, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 1, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}}, {{0, -1, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}} }
2457                     };
2458                     PetscReal normal[4];
2459                     PetscInt  e, f, g;
2460 
2461                     for (d = 0; d < embedDim; ++d) {
2462                       normal[d] = 0.0;
2463                       for (e = 0; e < embedDim; ++e) {
2464                         for (f = 0; f < embedDim; ++f) {
2465                           for (g = 0; g < embedDim; ++g) {
2466                             normal[d] += epsilon[d][e][f][g] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]) * (coordsIn[l * embedDim + f] - coordsIn[i * embedDim + f]);
2467                           }
2468                         }
2469                       }
2470                     }
2471                     if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2472                       PetscInt tmp = cone[1];
2473                       cone[1]      = cone[2];
2474                       cone[2]      = tmp;
2475                     }
2476                   }
2477                   PetscCall(DMPlexSetCone(dm, c++, cone));
2478                 }
2479               }
2480             }
2481           }
2482         }
2483       }
2484       PetscCall(DMPlexSymmetrize(dm));
2485       PetscCall(DMPlexStratify(dm));
2486       PetscCall(PetscFree(graph));
2487     }
2488     break;
2489   default:
2490     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unsupported dimension for sphere: %" PetscInt_FMT, dim);
2491   }
2492   /* Create coordinates */
2493   PetscCall(DMGetCoordinateSection(dm, &coordSection));
2494   PetscCall(PetscSectionSetNumFields(coordSection, 1));
2495   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, embedDim));
2496   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVerts));
2497   for (v = firstVertex; v < firstVertex + numVerts; ++v) {
2498     PetscCall(PetscSectionSetDof(coordSection, v, embedDim));
2499     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, embedDim));
2500   }
2501   PetscCall(PetscSectionSetUp(coordSection));
2502   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2503   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2504   PetscCall(VecSetBlockSize(coordinates, embedDim));
2505   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2506   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2507   PetscCall(VecSetType(coordinates, VECSTANDARD));
2508   PetscCall(VecGetArray(coordinates, &coords));
2509   for (v = 0; v < numVerts; ++v)
2510     for (d = 0; d < embedDim; ++d) coords[v * embedDim + d] = coordsIn[v * embedDim + d];
2511   PetscCall(VecRestoreArray(coordinates, &coords));
2512   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2513   PetscCall(VecDestroy(&coordinates));
2514   PetscCall(PetscFree(coordsIn));
2515   {
2516     DM          cdm;
2517     PetscDS     cds;
2518     PetscScalar c = R;
2519 
2520     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToSphere));
2521     PetscCall(DMGetCoordinateDM(dm, &cdm));
2522     PetscCall(DMGetDS(cdm, &cds));
2523     PetscCall(PetscDSSetConstants(cds, 1, &c));
2524   }
2525   /* Wait for coordinate creation before doing in-place modification */
2526   if (simplex) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2527   PetscFunctionReturn(0);
2528 }
2529 
2530 typedef void (*TPSEvaluateFunc)(const PetscReal[], PetscReal *, PetscReal[], PetscReal (*)[3]);
2531 
2532 /*
2533  The Schwarz P implicit surface is
2534 
2535      f(x) = cos(x0) + cos(x1) + cos(x2) = 0
2536 */
2537 static void TPSEvaluate_SchwarzP(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2538 {
2539   PetscReal c[3] = {PetscCosReal(y[0] * PETSC_PI), PetscCosReal(y[1] * PETSC_PI), PetscCosReal(y[2] * PETSC_PI)};
2540   PetscReal g[3] = {-PetscSinReal(y[0] * PETSC_PI), -PetscSinReal(y[1] * PETSC_PI), -PetscSinReal(y[2] * PETSC_PI)};
2541   f[0]           = c[0] + c[1] + c[2];
2542   for (PetscInt i = 0; i < 3; i++) {
2543     grad[i] = PETSC_PI * g[i];
2544     for (PetscInt j = 0; j < 3; j++) hess[i][j] = (i == j) ? -PetscSqr(PETSC_PI) * c[i] : 0.;
2545   }
2546 }
2547 
2548 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2549 static PetscErrorCode TPSExtrudeNormalFunc_SchwarzP(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2550 {
2551   for (PetscInt i = 0; i < 3; i++) u[i] = -PETSC_PI * PetscSinReal(x[i] * PETSC_PI);
2552   return 0;
2553 }
2554 
2555 /*
2556  The Gyroid implicit surface is
2557 
2558  f(x,y,z) = sin(pi * x) * cos (pi * (y + 1/2))  + sin(pi * (y + 1/2)) * cos(pi * (z + 1/4)) + sin(pi * (z + 1/4)) * cos(pi * x)
2559 
2560 */
2561 static void TPSEvaluate_Gyroid(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2562 {
2563   PetscReal s[3] = {PetscSinReal(PETSC_PI * y[0]), PetscSinReal(PETSC_PI * (y[1] + .5)), PetscSinReal(PETSC_PI * (y[2] + .25))};
2564   PetscReal c[3] = {PetscCosReal(PETSC_PI * y[0]), PetscCosReal(PETSC_PI * (y[1] + .5)), PetscCosReal(PETSC_PI * (y[2] + .25))};
2565   f[0]           = s[0] * c[1] + s[1] * c[2] + s[2] * c[0];
2566   grad[0]        = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2567   grad[1]        = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2568   grad[2]        = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2569   hess[0][0]     = -PetscSqr(PETSC_PI) * (s[0] * c[1] + s[2] * c[0]);
2570   hess[0][1]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2571   hess[0][2]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2572   hess[1][0]     = -PetscSqr(PETSC_PI) * (s[1] * c[2] + s[0] * c[1]);
2573   hess[1][1]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2574   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2575   hess[2][0]     = -PetscSqr(PETSC_PI) * (s[2] * c[0] + s[1] * c[2]);
2576   hess[2][1]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2577   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2578 }
2579 
2580 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2581 static PetscErrorCode TPSExtrudeNormalFunc_Gyroid(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2582 {
2583   PetscReal s[3] = {PetscSinReal(PETSC_PI * x[0]), PetscSinReal(PETSC_PI * (x[1] + .5)), PetscSinReal(PETSC_PI * (x[2] + .25))};
2584   PetscReal c[3] = {PetscCosReal(PETSC_PI * x[0]), PetscCosReal(PETSC_PI * (x[1] + .5)), PetscCosReal(PETSC_PI * (x[2] + .25))};
2585   u[0]           = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2586   u[1]           = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2587   u[2]           = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2588   return 0;
2589 }
2590 
2591 /*
2592    We wish to solve
2593 
2594          min_y || y - x ||^2  subject to f(y) = 0
2595 
2596    Let g(y) = grad(f).  The minimization problem is equivalent to asking to satisfy
2597    f(y) = 0 and (y-x) is parallel to g(y).  We do this by using Householder QR to obtain a basis for the
2598    tangent space and ask for both components in the tangent space to be zero.
2599 
2600    Take g to be a column vector and compute the "full QR" factorization Q R = g,
2601    where Q = I - 2 n n^T is a symmetric orthogonal matrix.
2602    The first column of Q is parallel to g so the remaining two columns span the null space.
2603    Let Qn = Q[:,1:] be those remaining columns.  Then Qn Qn^T is an orthogonal projector into the tangent space.
2604    Since Q is symmetric, this is equivalent to multipyling by Q and taking the last two entries.
2605    In total, we have a system of 3 equations in 3 unknowns:
2606 
2607      f(y) = 0                       1 equation
2608      Qn^T (y - x) = 0               2 equations
2609 
2610    Here, we compute the residual and Jacobian of this system.
2611 */
2612 static void TPSNearestPointResJac(TPSEvaluateFunc feval, const PetscScalar x[], const PetscScalar y[], PetscScalar res[], PetscScalar J[])
2613 {
2614   PetscReal yreal[3] = {PetscRealPart(y[0]), PetscRealPart(y[1]), PetscRealPart(y[2])};
2615   PetscReal d[3]     = {PetscRealPart(y[0] - x[0]), PetscRealPart(y[1] - x[1]), PetscRealPart(y[2] - x[2])};
2616   PetscReal f, grad[3], n[3], norm, norm_y[3], nd, nd_y[3], sign;
2617   PetscReal n_y[3][3] = {
2618     {0, 0, 0},
2619     {0, 0, 0},
2620     {0, 0, 0}
2621   };
2622 
2623   feval(yreal, &f, grad, n_y);
2624 
2625   for (PetscInt i = 0; i < 3; i++) n[i] = grad[i];
2626   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2627   for (PetscInt i = 0; i < 3; i++) norm_y[i] = 1. / norm * n[i] * n_y[i][i];
2628 
2629   // Define the Householder reflector
2630   sign = n[0] >= 0 ? 1. : -1.;
2631   n[0] += norm * sign;
2632   for (PetscInt i = 0; i < 3; i++) n_y[0][i] += norm_y[i] * sign;
2633 
2634   norm      = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2635   norm_y[0] = 1. / norm * (n[0] * n_y[0][0]);
2636   norm_y[1] = 1. / norm * (n[0] * n_y[0][1] + n[1] * n_y[1][1]);
2637   norm_y[2] = 1. / norm * (n[0] * n_y[0][2] + n[2] * n_y[2][2]);
2638 
2639   for (PetscInt i = 0; i < 3; i++) {
2640     n[i] /= norm;
2641     for (PetscInt j = 0; j < 3; j++) {
2642       // note that n[i] is n_old[i]/norm when executing the code below
2643       n_y[i][j] = n_y[i][j] / norm - n[i] / norm * norm_y[j];
2644     }
2645   }
2646 
2647   nd = n[0] * d[0] + n[1] * d[1] + n[2] * d[2];
2648   for (PetscInt i = 0; i < 3; i++) nd_y[i] = n[i] + n_y[0][i] * d[0] + n_y[1][i] * d[1] + n_y[2][i] * d[2];
2649 
2650   res[0] = f;
2651   res[1] = d[1] - 2 * n[1] * nd;
2652   res[2] = d[2] - 2 * n[2] * nd;
2653   // J[j][i] is J_{ij} (column major)
2654   for (PetscInt j = 0; j < 3; j++) {
2655     J[0 + j * 3] = grad[j];
2656     J[1 + j * 3] = (j == 1) * 1. - 2 * (n_y[1][j] * nd + n[1] * nd_y[j]);
2657     J[2 + j * 3] = (j == 2) * 1. - 2 * (n_y[2][j] * nd + n[2] * nd_y[j]);
2658   }
2659 }
2660 
2661 /*
2662    Project x to the nearest point on the implicit surface using Newton's method.
2663 */
2664 static PetscErrorCode TPSNearestPoint(TPSEvaluateFunc feval, PetscScalar x[])
2665 {
2666   PetscScalar y[3] = {x[0], x[1], x[2]}; // Initial guess
2667 
2668   PetscFunctionBegin;
2669   for (PetscInt iter = 0; iter < 10; iter++) {
2670     PetscScalar res[3], J[9];
2671     PetscReal   resnorm;
2672     TPSNearestPointResJac(feval, x, y, res, J);
2673     resnorm = PetscSqrtReal(PetscSqr(PetscRealPart(res[0])) + PetscSqr(PetscRealPart(res[1])) + PetscSqr(PetscRealPart(res[2])));
2674     if (0) { // Turn on this monitor if you need to confirm quadratic convergence
2675       PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%" PetscInt_FMT "] res [%g %g %g]\n", iter, (double)PetscRealPart(res[0]), (double)PetscRealPart(res[1]), (double)PetscRealPart(res[2])));
2676     }
2677     if (resnorm < PETSC_SMALL) break;
2678 
2679     // Take the Newton step
2680     PetscCall(PetscKernel_A_gets_inverse_A_3(J, 0., PETSC_FALSE, NULL));
2681     PetscKernel_v_gets_v_minus_A_times_w_3(y, J, res);
2682   }
2683   for (PetscInt i = 0; i < 3; i++) x[i] = y[i];
2684   PetscFunctionReturn(0);
2685 }
2686 
2687 const char *const DMPlexTPSTypes[] = {"SCHWARZ_P", "GYROID", "DMPlexTPSType", "DMPLEX_TPS_", NULL};
2688 
2689 static PetscErrorCode DMPlexCreateTPSMesh_Internal(DM dm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness)
2690 {
2691   PetscMPIInt rank;
2692   PetscInt    topoDim = 2, spaceDim = 3, numFaces = 0, numVertices = 0, numEdges = 0;
2693   PetscInt(*edges)[2] = NULL, *edgeSets = NULL;
2694   PetscInt            *cells_flat = NULL;
2695   PetscReal           *vtxCoords  = NULL;
2696   TPSEvaluateFunc      evalFunc   = NULL;
2697   PetscSimplePointFunc normalFunc = NULL;
2698   DMLabel              label;
2699 
2700   PetscFunctionBegin;
2701   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2702   PetscCheck((layers != 0) ^ (thickness == 0.), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Layers %" PetscInt_FMT " must be nonzero iff thickness %g is nonzero", layers, (double)thickness);
2703   switch (tpstype) {
2704   case DMPLEX_TPS_SCHWARZ_P:
2705     PetscCheck(!periodic || (periodic[0] == DM_BOUNDARY_NONE && periodic[1] == DM_BOUNDARY_NONE && periodic[2] == DM_BOUNDARY_NONE), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Schwarz P does not support periodic meshes");
2706     if (rank == 0) {
2707       PetscInt(*cells)[6][4][4] = NULL; // [junction, junction-face, cell, conn]
2708       PetscInt  Njunctions = 0, Ncuts = 0, Npipes[3], vcount;
2709       PetscReal L = 1;
2710 
2711       Npipes[0]   = (extent[0] + 1) * extent[1] * extent[2];
2712       Npipes[1]   = extent[0] * (extent[1] + 1) * extent[2];
2713       Npipes[2]   = extent[0] * extent[1] * (extent[2] + 1);
2714       Njunctions  = extent[0] * extent[1] * extent[2];
2715       Ncuts       = 2 * (extent[0] * extent[1] + extent[1] * extent[2] + extent[2] * extent[0]);
2716       numVertices = 4 * (Npipes[0] + Npipes[1] + Npipes[2]) + 8 * Njunctions;
2717       PetscCall(PetscMalloc1(3 * numVertices, &vtxCoords));
2718       PetscCall(PetscMalloc1(Njunctions, &cells));
2719       PetscCall(PetscMalloc1(Ncuts * 4, &edges));
2720       PetscCall(PetscMalloc1(Ncuts * 4, &edgeSets));
2721       // x-normal pipes
2722       vcount = 0;
2723       for (PetscInt i = 0; i < extent[0] + 1; i++) {
2724         for (PetscInt j = 0; j < extent[1]; j++) {
2725           for (PetscInt k = 0; k < extent[2]; k++) {
2726             for (PetscInt l = 0; l < 4; l++) {
2727               vtxCoords[vcount++] = (2 * i - 1) * L;
2728               vtxCoords[vcount++] = 2 * j * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2729               vtxCoords[vcount++] = 2 * k * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2730             }
2731           }
2732         }
2733       }
2734       // y-normal pipes
2735       for (PetscInt i = 0; i < extent[0]; i++) {
2736         for (PetscInt j = 0; j < extent[1] + 1; j++) {
2737           for (PetscInt k = 0; k < extent[2]; k++) {
2738             for (PetscInt l = 0; l < 4; l++) {
2739               vtxCoords[vcount++] = 2 * i * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2740               vtxCoords[vcount++] = (2 * j - 1) * L;
2741               vtxCoords[vcount++] = 2 * k * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2742             }
2743           }
2744         }
2745       }
2746       // z-normal pipes
2747       for (PetscInt i = 0; i < extent[0]; i++) {
2748         for (PetscInt j = 0; j < extent[1]; j++) {
2749           for (PetscInt k = 0; k < extent[2] + 1; k++) {
2750             for (PetscInt l = 0; l < 4; l++) {
2751               vtxCoords[vcount++] = 2 * i * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2752               vtxCoords[vcount++] = 2 * j * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2753               vtxCoords[vcount++] = (2 * k - 1) * L;
2754             }
2755           }
2756         }
2757       }
2758       // junctions
2759       for (PetscInt i = 0; i < extent[0]; i++) {
2760         for (PetscInt j = 0; j < extent[1]; j++) {
2761           for (PetscInt k = 0; k < extent[2]; k++) {
2762             const PetscInt J = (i * extent[1] + j) * extent[2] + k, Jvoff = (Npipes[0] + Npipes[1] + Npipes[2]) * 4 + J * 8;
2763             PetscCheck(vcount / 3 == Jvoff, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected vertex count");
2764             for (PetscInt ii = 0; ii < 2; ii++) {
2765               for (PetscInt jj = 0; jj < 2; jj++) {
2766                 for (PetscInt kk = 0; kk < 2; kk++) {
2767                   double Ls           = (1 - sqrt(2) / 4) * L;
2768                   vtxCoords[vcount++] = 2 * i * L + (2 * ii - 1) * Ls;
2769                   vtxCoords[vcount++] = 2 * j * L + (2 * jj - 1) * Ls;
2770                   vtxCoords[vcount++] = 2 * k * L + (2 * kk - 1) * Ls;
2771                 }
2772               }
2773             }
2774             const PetscInt jfaces[3][2][4] = {
2775               {{3, 1, 0, 2}, {7, 5, 4, 6}}, // x-aligned
2776               {{5, 4, 0, 1}, {7, 6, 2, 3}}, // y-aligned
2777               {{6, 2, 0, 4}, {7, 3, 1, 5}}  // z-aligned
2778             };
2779             const PetscInt pipe_lo[3] = {// vertex numbers of pipes
2780                                          ((i * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + Npipes[0] + Npipes[1]) * 4};
2781             const PetscInt pipe_hi[3] = {// vertex numbers of pipes
2782                                          (((i + 1) * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j + 1) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + 1 + Npipes[0] + Npipes[1]) * 4};
2783             for (PetscInt dir = 0; dir < 3; dir++) { // x,y,z
2784               const PetscInt ijk[3] = {i, j, k};
2785               for (PetscInt l = 0; l < 4; l++) { // rotations
2786                 cells[J][dir * 2 + 0][l][0] = pipe_lo[dir] + l;
2787                 cells[J][dir * 2 + 0][l][1] = Jvoff + jfaces[dir][0][l];
2788                 cells[J][dir * 2 + 0][l][2] = Jvoff + jfaces[dir][0][(l - 1 + 4) % 4];
2789                 cells[J][dir * 2 + 0][l][3] = pipe_lo[dir] + (l - 1 + 4) % 4;
2790                 cells[J][dir * 2 + 1][l][0] = Jvoff + jfaces[dir][1][l];
2791                 cells[J][dir * 2 + 1][l][1] = pipe_hi[dir] + l;
2792                 cells[J][dir * 2 + 1][l][2] = pipe_hi[dir] + (l - 1 + 4) % 4;
2793                 cells[J][dir * 2 + 1][l][3] = Jvoff + jfaces[dir][1][(l - 1 + 4) % 4];
2794                 if (ijk[dir] == 0) {
2795                   edges[numEdges][0] = pipe_lo[dir] + l;
2796                   edges[numEdges][1] = pipe_lo[dir] + (l + 1) % 4;
2797                   edgeSets[numEdges] = dir * 2 + 1;
2798                   numEdges++;
2799                 }
2800                 if (ijk[dir] + 1 == extent[dir]) {
2801                   edges[numEdges][0] = pipe_hi[dir] + l;
2802                   edges[numEdges][1] = pipe_hi[dir] + (l + 1) % 4;
2803                   edgeSets[numEdges] = dir * 2 + 2;
2804                   numEdges++;
2805                 }
2806               }
2807             }
2808           }
2809         }
2810       }
2811       PetscCheck(numEdges == Ncuts * 4, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Edge count %" PetscInt_FMT " incompatible with number of cuts %" PetscInt_FMT, numEdges, Ncuts);
2812       numFaces   = 24 * Njunctions;
2813       cells_flat = cells[0][0][0];
2814     }
2815     evalFunc   = TPSEvaluate_SchwarzP;
2816     normalFunc = TPSExtrudeNormalFunc_SchwarzP;
2817     break;
2818   case DMPLEX_TPS_GYROID:
2819     if (rank == 0) {
2820       // This is a coarse mesh approximation of the gyroid shifted to being the zero of the level set
2821       //
2822       //     sin(pi*x)*cos(pi*(y+1/2)) + sin(pi*(y+1/2))*cos(pi*(z+1/4)) + sin(pi*(z+1/4))*cos(x)
2823       //
2824       // on the cell [0,2]^3.
2825       //
2826       // Think about dividing that cell into four columns, and focus on the column [0,1]x[0,1]x[0,2].
2827       // If you looked at the gyroid in that column at different slices of z you would see that it kind of spins
2828       // like a boomerang:
2829       //
2830       //     z = 0          z = 1/4        z = 1/2        z = 3/4     //
2831       //     -----          -------        -------        -------     //
2832       //                                                              //
2833       //     +       +      +       +      +       +      +   \   +   //
2834       //      \                                   /            \      //
2835       //       \            `-_   _-'            /              }     //
2836       //        *-_            `-'            _-'              /      //
2837       //     +     `-+      +       +      +-'     +      +   /   +   //
2838       //                                                              //
2839       //                                                              //
2840       //     z = 1          z = 5/4        z = 3/2        z = 7/4     //
2841       //     -----          -------        -------        -------     //
2842       //                                                              //
2843       //     +-_     +      +       +      +     _-+      +   /   +   //
2844       //        `-_            _-_            _-`            /        //
2845       //           \        _-'   `-_        /              {         //
2846       //            \                       /                \        //
2847       //     +       +      +       +      +       +      +   \   +   //
2848       //
2849       //
2850       // This course mesh approximates each of these slices by two line segments,
2851       // and then connects the segments in consecutive layers with quadrilateral faces.
2852       // All of the end points of the segments are multiples of 1/4 except for the
2853       // point * in the picture for z = 0 above and the similar points in other layers.
2854       // That point is at (gamma, gamma, 0), where gamma is calculated below.
2855       //
2856       // The column  [1,2]x[1,2]x[0,2] looks the same as this column;
2857       // The columns [1,2]x[0,1]x[0,2] and [0,1]x[1,2]x[0,2] are mirror images.
2858       //
2859       // As for how this method turned into the names given to the vertices:
2860       // that was not systematic, it was just the way it worked out in my handwritten notes.
2861 
2862       PetscInt facesPerBlock = 64;
2863       PetscInt vertsPerBlock = 56;
2864       PetscInt extentPlus[3];
2865       PetscInt numBlocks, numBlocksPlus;
2866       const PetscInt A = 0, B = 1, C = 2, D = 3, E = 4, F = 5, G = 6, H = 7, II = 8, J = 9, K = 10, L = 11, M = 12, N = 13, O = 14, P = 15, Q = 16, R = 17, S = 18, T = 19, U = 20, V = 21, W = 22, X = 23, Y = 24, Z = 25, Ap = 26, Bp = 27, Cp = 28, Dp = 29, Ep = 30, Fp = 31, Gp = 32, Hp = 33, Ip = 34, Jp = 35, Kp = 36, Lp = 37, Mp = 38, Np = 39, Op = 40, Pp = 41, Qp = 42, Rp = 43, Sp = 44, Tp = 45, Up = 46, Vp = 47, Wp = 48, Xp = 49, Yp = 50, Zp = 51, Aq = 52, Bq = 53, Cq = 54, Dq = 55;
2867       const PetscInt pattern[64][4] = {
2868   /* face to vertex within the coarse discretization of a single gyroid block */
2869   /* layer 0 */
2870         {A,           C,           K,           G          },
2871         {C,           B,           II,          K          },
2872         {D,           A,           H,           L          },
2873         {B + 56 * 1,  D,           L,           J          },
2874         {E,           B + 56 * 1,  J,           N          },
2875         {A + 56 * 2,  E,           N,           H + 56 * 2 },
2876         {F,           A + 56 * 2,  G + 56 * 2,  M          },
2877         {B,           F,           M,           II         },
2878  /* layer 1 */
2879         {G,           K,           Q,           O          },
2880         {K,           II,          P,           Q          },
2881         {L,           H,           O + 56 * 1,  R          },
2882         {J,           L,           R,           P          },
2883         {N,           J,           P,           S          },
2884         {H + 56 * 2,  N,           S,           O + 56 * 3 },
2885         {M,           G + 56 * 2,  O + 56 * 2,  T          },
2886         {II,          M,           T,           P          },
2887  /* layer 2 */
2888         {O,           Q,           Y,           U          },
2889         {Q,           P,           W,           Y          },
2890         {R,           O + 56 * 1,  U + 56 * 1,  Ap         },
2891         {P,           R,           Ap,          W          },
2892         {S,           P,           X,           Bp         },
2893         {O + 56 * 3,  S,           Bp,          V + 56 * 1 },
2894         {T,           O + 56 * 2,  V,           Z          },
2895         {P,           T,           Z,           X          },
2896  /* layer 3 */
2897         {U,           Y,           Ep,          Dp         },
2898         {Y,           W,           Cp,          Ep         },
2899         {Ap,          U + 56 * 1,  Dp + 56 * 1, Gp         },
2900         {W,           Ap,          Gp,          Cp         },
2901         {Bp,          X,           Cp + 56 * 2, Fp         },
2902         {V + 56 * 1,  Bp,          Fp,          Dp + 56 * 1},
2903         {Z,           V,           Dp,          Hp         },
2904         {X,           Z,           Hp,          Cp + 56 * 2},
2905  /* layer 4 */
2906         {Dp,          Ep,          Mp,          Kp         },
2907         {Ep,          Cp,          Ip,          Mp         },
2908         {Gp,          Dp + 56 * 1, Lp,          Np         },
2909         {Cp,          Gp,          Np,          Jp         },
2910         {Fp,          Cp + 56 * 2, Jp + 56 * 2, Pp         },
2911         {Dp + 56 * 1, Fp,          Pp,          Lp         },
2912         {Hp,          Dp,          Kp,          Op         },
2913         {Cp + 56 * 2, Hp,          Op,          Ip + 56 * 2},
2914  /* layer 5 */
2915         {Kp,          Mp,          Sp,          Rp         },
2916         {Mp,          Ip,          Qp,          Sp         },
2917         {Np,          Lp,          Rp,          Tp         },
2918         {Jp,          Np,          Tp,          Qp + 56 * 1},
2919         {Pp,          Jp + 56 * 2, Qp + 56 * 3, Up         },
2920         {Lp,          Pp,          Up,          Rp         },
2921         {Op,          Kp,          Rp,          Vp         },
2922         {Ip + 56 * 2, Op,          Vp,          Qp + 56 * 2},
2923  /* layer 6 */
2924         {Rp,          Sp,          Aq,          Yp         },
2925         {Sp,          Qp,          Wp,          Aq         },
2926         {Tp,          Rp,          Yp,          Cq         },
2927         {Qp + 56 * 1, Tp,          Cq,          Wp + 56 * 1},
2928         {Up,          Qp + 56 * 3, Xp + 56 * 1, Dq         },
2929         {Rp,          Up,          Dq,          Zp         },
2930         {Vp,          Rp,          Zp,          Bq         },
2931         {Qp + 56 * 2, Vp,          Bq,          Xp         },
2932  /* layer 7 (the top is the periodic image of the bottom of layer 0) */
2933         {Yp,          Aq,          C + 56 * 4,  A + 56 * 4 },
2934         {Aq,          Wp,          B + 56 * 4,  C + 56 * 4 },
2935         {Cq,          Yp,          A + 56 * 4,  D + 56 * 4 },
2936         {Wp + 56 * 1, Cq,          D + 56 * 4,  B + 56 * 5 },
2937         {Dq,          Xp + 56 * 1, B + 56 * 5,  E + 56 * 4 },
2938         {Zp,          Dq,          E + 56 * 4,  A + 56 * 6 },
2939         {Bq,          Zp,          A + 56 * 6,  F + 56 * 4 },
2940         {Xp,          Bq,          F + 56 * 4,  B + 56 * 4 }
2941       };
2942       const PetscReal gamma                = PetscAcosReal((PetscSqrtReal(3.) - 1.) / PetscSqrtReal(2.)) / PETSC_PI;
2943       const PetscReal patternCoords[56][3] = {
2944         {1.,        0.,        0.  }, /* A  */
2945         {0.,        1.,        0.  }, /* B  */
2946         {gamma,     gamma,     0.  }, /* C  */
2947         {1 + gamma, 1 - gamma, 0.  }, /* D  */
2948         {2 - gamma, 2 - gamma, 0.  }, /* E  */
2949         {1 - gamma, 1 + gamma, 0.  }, /* F  */
2950 
2951         {.5,        0,         .25 }, /* G  */
2952         {1.5,       0.,        .25 }, /* H  */
2953         {.5,        1.,        .25 }, /* II */
2954         {1.5,       1.,        .25 }, /* J  */
2955         {.25,       .5,        .25 }, /* K  */
2956         {1.25,      .5,        .25 }, /* L  */
2957         {.75,       1.5,       .25 }, /* M  */
2958         {1.75,      1.5,       .25 }, /* N  */
2959 
2960         {0.,        0.,        .5  }, /* O  */
2961         {1.,        1.,        .5  }, /* P  */
2962         {gamma,     1 - gamma, .5  }, /* Q  */
2963         {1 + gamma, gamma,     .5  }, /* R  */
2964         {2 - gamma, 1 + gamma, .5  }, /* S  */
2965         {1 - gamma, 2 - gamma, .5  }, /* T  */
2966 
2967         {0.,        .5,        .75 }, /* U  */
2968         {0.,        1.5,       .75 }, /* V  */
2969         {1.,        .5,        .75 }, /* W  */
2970         {1.,        1.5,       .75 }, /* X  */
2971         {.5,        .75,       .75 }, /* Y  */
2972         {.5,        1.75,      .75 }, /* Z  */
2973         {1.5,       .25,       .75 }, /* Ap */
2974         {1.5,       1.25,      .75 }, /* Bp */
2975 
2976         {1.,        0.,        1.  }, /* Cp */
2977         {0.,        1.,        1.  }, /* Dp */
2978         {1 - gamma, 1 - gamma, 1.  }, /* Ep */
2979         {1 + gamma, 1 + gamma, 1.  }, /* Fp */
2980         {2 - gamma, gamma,     1.  }, /* Gp */
2981         {gamma,     2 - gamma, 1.  }, /* Hp */
2982 
2983         {.5,        0.,        1.25}, /* Ip */
2984         {1.5,       0.,        1.25}, /* Jp */
2985         {.5,        1.,        1.25}, /* Kp */
2986         {1.5,       1.,        1.25}, /* Lp */
2987         {.75,       .5,        1.25}, /* Mp */
2988         {1.75,      .5,        1.25}, /* Np */
2989         {.25,       1.5,       1.25}, /* Op */
2990         {1.25,      1.5,       1.25}, /* Pp */
2991 
2992         {0.,        0.,        1.5 }, /* Qp */
2993         {1.,        1.,        1.5 }, /* Rp */
2994         {1 - gamma, gamma,     1.5 }, /* Sp */
2995         {2 - gamma, 1 - gamma, 1.5 }, /* Tp */
2996         {1 + gamma, 2 - gamma, 1.5 }, /* Up */
2997         {gamma,     1 + gamma, 1.5 }, /* Vp */
2998 
2999         {0.,        .5,        1.75}, /* Wp */
3000         {0.,        1.5,       1.75}, /* Xp */
3001         {1.,        .5,        1.75}, /* Yp */
3002         {1.,        1.5,       1.75}, /* Zp */
3003         {.5,        .25,       1.75}, /* Aq */
3004         {.5,        1.25,      1.75}, /* Bq */
3005         {1.5,       .75,       1.75}, /* Cq */
3006         {1.5,       1.75,      1.75}, /* Dq */
3007       };
3008       PetscInt(*cells)[64][4] = NULL;
3009       PetscBool *seen;
3010       PetscInt  *vertToTrueVert;
3011       PetscInt   count;
3012 
3013       for (PetscInt i = 0; i < 3; i++) extentPlus[i] = extent[i] + 1;
3014       numBlocks = 1;
3015       for (PetscInt i = 0; i < 3; i++) numBlocks *= extent[i];
3016       numBlocksPlus = 1;
3017       for (PetscInt i = 0; i < 3; i++) numBlocksPlus *= extentPlus[i];
3018       numFaces = numBlocks * facesPerBlock;
3019       PetscCall(PetscMalloc1(numBlocks, &cells));
3020       PetscCall(PetscCalloc1(numBlocksPlus * vertsPerBlock, &seen));
3021       for (PetscInt k = 0; k < extent[2]; k++) {
3022         for (PetscInt j = 0; j < extent[1]; j++) {
3023           for (PetscInt i = 0; i < extent[0]; i++) {
3024             for (PetscInt f = 0; f < facesPerBlock; f++) {
3025               for (PetscInt v = 0; v < 4; v++) {
3026                 PetscInt vertRaw     = pattern[f][v];
3027                 PetscInt blockidx    = vertRaw / 56;
3028                 PetscInt patternvert = vertRaw % 56;
3029                 PetscInt xplus       = (blockidx & 1);
3030                 PetscInt yplus       = (blockidx & 2) >> 1;
3031                 PetscInt zplus       = (blockidx & 4) >> 2;
3032                 PetscInt zcoord      = (periodic && periodic[2] == DM_BOUNDARY_PERIODIC) ? ((k + zplus) % extent[2]) : (k + zplus);
3033                 PetscInt ycoord      = (periodic && periodic[1] == DM_BOUNDARY_PERIODIC) ? ((j + yplus) % extent[1]) : (j + yplus);
3034                 PetscInt xcoord      = (periodic && periodic[0] == DM_BOUNDARY_PERIODIC) ? ((i + xplus) % extent[0]) : (i + xplus);
3035                 PetscInt vert        = ((zcoord * extentPlus[1] + ycoord) * extentPlus[0] + xcoord) * 56 + patternvert;
3036 
3037                 cells[(k * extent[1] + j) * extent[0] + i][f][v] = vert;
3038                 seen[vert]                                       = PETSC_TRUE;
3039               }
3040             }
3041           }
3042         }
3043       }
3044       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++)
3045         if (seen[i]) numVertices++;
3046       count = 0;
3047       PetscCall(PetscMalloc1(numBlocksPlus * vertsPerBlock, &vertToTrueVert));
3048       PetscCall(PetscMalloc1(numVertices * 3, &vtxCoords));
3049       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) vertToTrueVert[i] = -1;
3050       for (PetscInt k = 0; k < extentPlus[2]; k++) {
3051         for (PetscInt j = 0; j < extentPlus[1]; j++) {
3052           for (PetscInt i = 0; i < extentPlus[0]; i++) {
3053             for (PetscInt v = 0; v < vertsPerBlock; v++) {
3054               PetscInt vIdx = ((k * extentPlus[1] + j) * extentPlus[0] + i) * vertsPerBlock + v;
3055 
3056               if (seen[vIdx]) {
3057                 PetscInt thisVert;
3058 
3059                 vertToTrueVert[vIdx] = thisVert = count++;
3060 
3061                 for (PetscInt d = 0; d < 3; d++) vtxCoords[3 * thisVert + d] = patternCoords[v][d];
3062                 vtxCoords[3 * thisVert + 0] += i * 2;
3063                 vtxCoords[3 * thisVert + 1] += j * 2;
3064                 vtxCoords[3 * thisVert + 2] += k * 2;
3065               }
3066             }
3067           }
3068         }
3069       }
3070       for (PetscInt i = 0; i < numBlocks; i++) {
3071         for (PetscInt f = 0; f < facesPerBlock; f++) {
3072           for (PetscInt v = 0; v < 4; v++) cells[i][f][v] = vertToTrueVert[cells[i][f][v]];
3073         }
3074       }
3075       PetscCall(PetscFree(vertToTrueVert));
3076       PetscCall(PetscFree(seen));
3077       cells_flat = cells[0][0];
3078       numEdges   = 0;
3079       for (PetscInt i = 0; i < numFaces; i++) {
3080         for (PetscInt e = 0; e < 4; e++) {
3081           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3082           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3083 
3084           for (PetscInt d = 0; d < 3; d++) {
3085             if (!periodic || periodic[0] != DM_BOUNDARY_PERIODIC) {
3086               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) numEdges++;
3087               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) numEdges++;
3088             }
3089           }
3090         }
3091       }
3092       PetscCall(PetscMalloc1(numEdges, &edges));
3093       PetscCall(PetscMalloc1(numEdges, &edgeSets));
3094       for (PetscInt edge = 0, i = 0; i < numFaces; i++) {
3095         for (PetscInt e = 0; e < 4; e++) {
3096           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3097           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3098 
3099           for (PetscInt d = 0; d < 3; d++) {
3100             if (!periodic || periodic[d] != DM_BOUNDARY_PERIODIC) {
3101               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) {
3102                 edges[edge][0]   = ev[0];
3103                 edges[edge][1]   = ev[1];
3104                 edgeSets[edge++] = 2 * d;
3105               }
3106               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) {
3107                 edges[edge][0]   = ev[0];
3108                 edges[edge][1]   = ev[1];
3109                 edgeSets[edge++] = 2 * d + 1;
3110               }
3111             }
3112           }
3113         }
3114       }
3115     }
3116     evalFunc   = TPSEvaluate_Gyroid;
3117     normalFunc = TPSExtrudeNormalFunc_Gyroid;
3118     break;
3119   }
3120 
3121   PetscCall(DMSetDimension(dm, topoDim));
3122   if (rank == 0) PetscCall(DMPlexBuildFromCellList(dm, numFaces, numVertices, 4, cells_flat));
3123   else PetscCall(DMPlexBuildFromCellList(dm, 0, 0, 0, NULL));
3124   PetscCall(PetscFree(cells_flat));
3125   {
3126     DM idm;
3127     PetscCall(DMPlexInterpolate(dm, &idm));
3128     PetscCall(DMPlexReplace_Internal(dm, &idm));
3129   }
3130   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, vtxCoords));
3131   else PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, NULL));
3132   PetscCall(PetscFree(vtxCoords));
3133 
3134   PetscCall(DMCreateLabel(dm, "Face Sets"));
3135   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3136   for (PetscInt e = 0; e < numEdges; e++) {
3137     PetscInt        njoin;
3138     const PetscInt *join, verts[] = {numFaces + edges[e][0], numFaces + edges[e][1]};
3139     PetscCall(DMPlexGetJoin(dm, 2, verts, &njoin, &join));
3140     PetscCheck(njoin == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected unique join of vertices %" PetscInt_FMT " and %" PetscInt_FMT, edges[e][0], edges[e][1]);
3141     PetscCall(DMLabelSetValue(label, join[0], edgeSets[e]));
3142     PetscCall(DMPlexRestoreJoin(dm, 2, verts, &njoin, &join));
3143   }
3144   PetscCall(PetscFree(edges));
3145   PetscCall(PetscFree(edgeSets));
3146   if (tps_distribute) {
3147     DM               pdm = NULL;
3148     PetscPartitioner part;
3149 
3150     PetscCall(DMPlexGetPartitioner(dm, &part));
3151     PetscCall(PetscPartitionerSetFromOptions(part));
3152     PetscCall(DMPlexDistribute(dm, 0, NULL, &pdm));
3153     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
3154     // Do not auto-distribute again
3155     PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
3156   }
3157 
3158   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3159   for (PetscInt refine = 0; refine < refinements; refine++) {
3160     PetscInt     m;
3161     DM           dmf;
3162     Vec          X;
3163     PetscScalar *x;
3164     PetscCall(DMRefine(dm, MPI_COMM_NULL, &dmf));
3165     PetscCall(DMPlexReplace_Internal(dm, &dmf));
3166 
3167     PetscCall(DMGetCoordinatesLocal(dm, &X));
3168     PetscCall(VecGetLocalSize(X, &m));
3169     PetscCall(VecGetArray(X, &x));
3170     for (PetscInt i = 0; i < m; i += 3) PetscCall(TPSNearestPoint(evalFunc, &x[i]));
3171     PetscCall(VecRestoreArray(X, &x));
3172   }
3173 
3174   // Face Sets has already been propagated to new vertices during refinement; this propagates to the initial vertices.
3175   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3176   PetscCall(DMPlexLabelComplete(dm, label));
3177 
3178   if (thickness > 0) {
3179     DM              edm, cdm, ecdm;
3180     DMPlexTransform tr;
3181     const char     *prefix;
3182     PetscOptions    options;
3183     // Code from DMPlexExtrude
3184     PetscCall(DMPlexTransformCreate(PetscObjectComm((PetscObject)dm), &tr));
3185     PetscCall(DMPlexTransformSetDM(tr, dm));
3186     PetscCall(DMPlexTransformSetType(tr, DMPLEXEXTRUDE));
3187     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
3188     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)tr, prefix));
3189     PetscCall(PetscObjectGetOptions((PetscObject)dm, &options));
3190     PetscCall(PetscObjectSetOptions((PetscObject)tr, options));
3191     PetscCall(DMPlexTransformExtrudeSetLayers(tr, layers));
3192     PetscCall(DMPlexTransformExtrudeSetThickness(tr, thickness));
3193     PetscCall(DMPlexTransformExtrudeSetTensor(tr, PETSC_FALSE));
3194     PetscCall(DMPlexTransformExtrudeSetSymmetric(tr, PETSC_TRUE));
3195     PetscCall(DMPlexTransformExtrudeSetNormalFunction(tr, normalFunc));
3196     PetscCall(DMPlexTransformSetFromOptions(tr));
3197     PetscCall(PetscObjectSetOptions((PetscObject)tr, NULL));
3198     PetscCall(DMPlexTransformSetUp(tr));
3199     PetscCall(PetscObjectViewFromOptions((PetscObject)tr, NULL, "-dm_plex_tps_transform_view"));
3200     PetscCall(DMPlexTransformApply(tr, dm, &edm));
3201     PetscCall(DMCopyDisc(dm, edm));
3202     PetscCall(DMGetCoordinateDM(dm, &cdm));
3203     PetscCall(DMGetCoordinateDM(edm, &ecdm));
3204     PetscCall(DMCopyDisc(cdm, ecdm));
3205     PetscCall(DMPlexTransformCreateDiscLabels(tr, edm));
3206     PetscCall(DMPlexTransformDestroy(&tr));
3207     if (edm) {
3208       ((DM_Plex *)edm->data)->printFEM    = ((DM_Plex *)dm->data)->printFEM;
3209       ((DM_Plex *)edm->data)->printL2     = ((DM_Plex *)dm->data)->printL2;
3210       ((DM_Plex *)edm->data)->printLocate = ((DM_Plex *)dm->data)->printLocate;
3211     }
3212     PetscCall(DMPlexReplace_Internal(dm, &edm));
3213   }
3214   PetscFunctionReturn(0);
3215 }
3216 
3217 /*@
3218   DMPlexCreateTPSMesh - Create a distributed, interpolated mesh of a triply-periodic surface
3219 
3220   Collective
3221 
3222   Input Parameters:
3223 + comm   - The communicator for the DM object
3224 . tpstype - Type of triply-periodic surface
3225 . extent - Array of length 3 containing number of periods in each direction
3226 . periodic - array of length 3 with periodicity, or NULL for non-periodic
3227 . tps_distribute - Distribute 2D manifold mesh prior to refinement and extrusion (more scalable)
3228 . refinements - Number of factor-of-2 refinements of 2D manifold mesh
3229 . layers - Number of cell layers extruded in normal direction
3230 - thickness - Thickness in normal direction
3231 
3232   Output Parameter:
3233 . dm  - The DM object
3234 
3235   Notes:
3236   This meshes the surface of the Schwarz P or Gyroid surfaces.  Schwarz P is is the simplest member of the triply-periodic minimal surfaces.
3237   https://en.wikipedia.org/wiki/Schwarz_minimal_surface#Schwarz_P_(%22Primitive%22) and can be cut with "clean" boundaries.
3238   The Gyroid (https://en.wikipedia.org/wiki/Gyroid) is another triply-periodic minimal surface with applications in additive manufacturing; it is much more difficult to "cut" since there are no planes of symmetry.
3239   Our implementation creates a very coarse mesh of the surface and refines (by 4-way splitting) as many times as requested.
3240   On each refinement, all vertices are projected to their nearest point on the surface.
3241   This projection could readily be extended to related surfaces.
3242 
3243   The face (edge) sets for the Schwarz P surface are numbered 1(-x), 2(+x), 3(-y), 4(+y), 5(-z), 6(+z).
3244   When the mesh is refined, "Face Sets" contain the new vertices (created during refinement).  Use DMPlexLabelComplete() to propagate to coarse-level vertices.
3245 
3246   References:
3247 . * - Maskery et al, Insights into the mechanical properties of several triply periodic minimal surface lattice structures made by polymer additive manufacturing, 2017. https://doi.org/10.1016/j.polymer.2017.11.049
3248 
3249   Developer Notes:
3250   The Gyroid mesh does not currently mark boundary sets.
3251 
3252   Level: beginner
3253 
3254 .seealso: `DMPlexCreateSphereMesh()`, `DMSetType()`, `DMCreate()`
3255 @*/
3256 PetscErrorCode DMPlexCreateTPSMesh(MPI_Comm comm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness, DM *dm)
3257 {
3258   PetscFunctionBegin;
3259   PetscCall(DMCreate(comm, dm));
3260   PetscCall(DMSetType(*dm, DMPLEX));
3261   PetscCall(DMPlexCreateTPSMesh_Internal(*dm, tpstype, extent, periodic, tps_distribute, refinements, layers, thickness));
3262   PetscFunctionReturn(0);
3263 }
3264 
3265 /*@
3266   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
3267 
3268   Collective
3269 
3270   Input Parameters:
3271 + comm    - The communicator for the DM object
3272 . dim     - The dimension
3273 . simplex - Use simplices, or tensor product cells
3274 - R       - The radius
3275 
3276   Output Parameter:
3277 . dm  - The DM object
3278 
3279   Level: beginner
3280 
3281 .seealso: `DMPlexCreateBallMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3282 @*/
3283 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscReal R, DM *dm)
3284 {
3285   PetscFunctionBegin;
3286   PetscValidPointer(dm, 5);
3287   PetscCall(DMCreate(comm, dm));
3288   PetscCall(DMSetType(*dm, DMPLEX));
3289   PetscCall(DMPlexCreateSphereMesh_Internal(*dm, dim, simplex, R));
3290   PetscFunctionReturn(0);
3291 }
3292 
3293 static PetscErrorCode DMPlexCreateBallMesh_Internal(DM dm, PetscInt dim, PetscReal R)
3294 {
3295   DM      sdm, vol;
3296   DMLabel bdlabel;
3297 
3298   PetscFunctionBegin;
3299   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
3300   PetscCall(DMSetType(sdm, DMPLEX));
3301   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sdm, "bd_"));
3302   PetscCall(DMPlexCreateSphereMesh_Internal(sdm, dim - 1, PETSC_TRUE, R));
3303   PetscCall(DMSetFromOptions(sdm));
3304   PetscCall(DMViewFromOptions(sdm, NULL, "-dm_view"));
3305   PetscCall(DMPlexGenerate(sdm, NULL, PETSC_TRUE, &vol));
3306   PetscCall(DMDestroy(&sdm));
3307   PetscCall(DMPlexReplace_Internal(dm, &vol));
3308   PetscCall(DMCreateLabel(dm, "marker"));
3309   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
3310   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
3311   PetscCall(DMPlexLabelComplete(dm, bdlabel));
3312   PetscFunctionReturn(0);
3313 }
3314 
3315 /*@
3316   DMPlexCreateBallMesh - Creates a simplex mesh on the d-dimensional ball, B^d.
3317 
3318   Collective
3319 
3320   Input Parameters:
3321 + comm  - The communicator for the DM object
3322 . dim   - The dimension
3323 - R     - The radius
3324 
3325   Output Parameter:
3326 . dm  - The DM object
3327 
3328   Options Database Keys:
3329 - bd_dm_refine - This will refine the surface mesh preserving the sphere geometry
3330 
3331   Level: beginner
3332 
3333 .seealso: `DMPlexCreateSphereMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3334 @*/
3335 PetscErrorCode DMPlexCreateBallMesh(MPI_Comm comm, PetscInt dim, PetscReal R, DM *dm)
3336 {
3337   PetscFunctionBegin;
3338   PetscCall(DMCreate(comm, dm));
3339   PetscCall(DMSetType(*dm, DMPLEX));
3340   PetscCall(DMPlexCreateBallMesh_Internal(*dm, dim, R));
3341   PetscFunctionReturn(0);
3342 }
3343 
3344 static PetscErrorCode DMPlexCreateReferenceCell_Internal(DM rdm, DMPolytopeType ct)
3345 {
3346   PetscFunctionBegin;
3347   switch (ct) {
3348   case DM_POLYTOPE_POINT: {
3349     PetscInt    numPoints[1]        = {1};
3350     PetscInt    coneSize[1]         = {0};
3351     PetscInt    cones[1]            = {0};
3352     PetscInt    coneOrientations[1] = {0};
3353     PetscScalar vertexCoords[1]     = {0.0};
3354 
3355     PetscCall(DMSetDimension(rdm, 0));
3356     PetscCall(DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3357   } break;
3358   case DM_POLYTOPE_SEGMENT: {
3359     PetscInt    numPoints[2]        = {2, 1};
3360     PetscInt    coneSize[3]         = {2, 0, 0};
3361     PetscInt    cones[2]            = {1, 2};
3362     PetscInt    coneOrientations[2] = {0, 0};
3363     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3364 
3365     PetscCall(DMSetDimension(rdm, 1));
3366     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3367   } break;
3368   case DM_POLYTOPE_POINT_PRISM_TENSOR: {
3369     PetscInt    numPoints[2]        = {2, 1};
3370     PetscInt    coneSize[3]         = {2, 0, 0};
3371     PetscInt    cones[2]            = {1, 2};
3372     PetscInt    coneOrientations[2] = {0, 0};
3373     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3374 
3375     PetscCall(DMSetDimension(rdm, 1));
3376     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3377   } break;
3378   case DM_POLYTOPE_TRIANGLE: {
3379     PetscInt    numPoints[2]        = {3, 1};
3380     PetscInt    coneSize[4]         = {3, 0, 0, 0};
3381     PetscInt    cones[3]            = {1, 2, 3};
3382     PetscInt    coneOrientations[3] = {0, 0, 0};
3383     PetscScalar vertexCoords[6]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0};
3384 
3385     PetscCall(DMSetDimension(rdm, 2));
3386     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3387   } break;
3388   case DM_POLYTOPE_QUADRILATERAL: {
3389     PetscInt    numPoints[2]        = {4, 1};
3390     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3391     PetscInt    cones[4]            = {1, 2, 3, 4};
3392     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3393     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0};
3394 
3395     PetscCall(DMSetDimension(rdm, 2));
3396     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3397   } break;
3398   case DM_POLYTOPE_SEG_PRISM_TENSOR: {
3399     PetscInt    numPoints[2]        = {4, 1};
3400     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3401     PetscInt    cones[4]            = {1, 2, 3, 4};
3402     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3403     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0};
3404 
3405     PetscCall(DMSetDimension(rdm, 2));
3406     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3407   } break;
3408   case DM_POLYTOPE_TETRAHEDRON: {
3409     PetscInt    numPoints[2]        = {4, 1};
3410     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3411     PetscInt    cones[4]            = {1, 2, 3, 4};
3412     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3413     PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0};
3414 
3415     PetscCall(DMSetDimension(rdm, 3));
3416     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3417   } break;
3418   case DM_POLYTOPE_HEXAHEDRON: {
3419     PetscInt    numPoints[2]        = {8, 1};
3420     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3421     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3422     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3423     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3424 
3425     PetscCall(DMSetDimension(rdm, 3));
3426     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3427   } break;
3428   case DM_POLYTOPE_TRI_PRISM: {
3429     PetscInt    numPoints[2]        = {6, 1};
3430     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3431     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3432     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3433     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3434 
3435     PetscCall(DMSetDimension(rdm, 3));
3436     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3437   } break;
3438   case DM_POLYTOPE_TRI_PRISM_TENSOR: {
3439     PetscInt    numPoints[2]        = {6, 1};
3440     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3441     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3442     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3443     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3444 
3445     PetscCall(DMSetDimension(rdm, 3));
3446     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3447   } break;
3448   case DM_POLYTOPE_QUAD_PRISM_TENSOR: {
3449     PetscInt    numPoints[2]        = {8, 1};
3450     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3451     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3452     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3453     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3454 
3455     PetscCall(DMSetDimension(rdm, 3));
3456     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3457   } break;
3458   case DM_POLYTOPE_PYRAMID: {
3459     PetscInt    numPoints[2]        = {5, 1};
3460     PetscInt    coneSize[6]         = {5, 0, 0, 0, 0, 0};
3461     PetscInt    cones[5]            = {1, 2, 3, 4, 5};
3462     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3463     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, 0.0, 0.0, 1.0};
3464 
3465     PetscCall(DMSetDimension(rdm, 3));
3466     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3467   } break;
3468   default:
3469     SETERRQ(PetscObjectComm((PetscObject)rdm), PETSC_ERR_ARG_WRONG, "Cannot create reference cell for cell type %s", DMPolytopeTypes[ct]);
3470   }
3471   {
3472     PetscInt Nv, v;
3473 
3474     /* Must create the celltype label here so that we do not automatically try to compute the types */
3475     PetscCall(DMCreateLabel(rdm, "celltype"));
3476     PetscCall(DMPlexSetCellType(rdm, 0, ct));
3477     PetscCall(DMPlexGetChart(rdm, NULL, &Nv));
3478     for (v = 1; v < Nv; ++v) PetscCall(DMPlexSetCellType(rdm, v, DM_POLYTOPE_POINT));
3479   }
3480   PetscCall(DMPlexInterpolateInPlace_Internal(rdm));
3481   PetscCall(PetscObjectSetName((PetscObject)rdm, DMPolytopeTypes[ct]));
3482   PetscFunctionReturn(0);
3483 }
3484 
3485 /*@
3486   DMPlexCreateReferenceCell - Create a DMPLEX with the appropriate FEM reference cell
3487 
3488   Collective
3489 
3490   Input Parameters:
3491 + comm - The communicator
3492 - ct   - The cell type of the reference cell
3493 
3494   Output Parameter:
3495 . refdm - The reference cell
3496 
3497   Level: intermediate
3498 
3499 .seealso: `DMPlexCreateReferenceCell()`, `DMPlexCreateBoxMesh()`
3500 @*/
3501 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, DMPolytopeType ct, DM *refdm)
3502 {
3503   PetscFunctionBegin;
3504   PetscCall(DMCreate(comm, refdm));
3505   PetscCall(DMSetType(*refdm, DMPLEX));
3506   PetscCall(DMPlexCreateReferenceCell_Internal(*refdm, ct));
3507   PetscFunctionReturn(0);
3508 }
3509 
3510 static PetscErrorCode DMPlexCreateBoundaryLabel_Private(DM dm, const char name[])
3511 {
3512   DM        plex;
3513   DMLabel   label;
3514   PetscBool hasLabel;
3515 
3516   PetscFunctionBegin;
3517   PetscCall(DMHasLabel(dm, name, &hasLabel));
3518   if (hasLabel) PetscFunctionReturn(0);
3519   PetscCall(DMCreateLabel(dm, name));
3520   PetscCall(DMGetLabel(dm, name, &label));
3521   PetscCall(DMConvert(dm, DMPLEX, &plex));
3522   PetscCall(DMPlexMarkBoundaryFaces(plex, 1, label));
3523   PetscCall(DMPlexLabelComplete(plex, label));
3524   PetscCall(DMDestroy(&plex));
3525   PetscFunctionReturn(0);
3526 }
3527 
3528 /*
3529   We use the last coordinate as the radius, the inner radius is lower[dim-1] and the outer radius is upper[dim-1]. Then we map the first coordinate around the circle.
3530 
3531     (x, y) -> (r, theta) = (x[1], (x[0] - lower[0]) * 2\pi/(upper[0] - lower[0]))
3532 */
3533 static void boxToAnnulus(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
3534 {
3535   const PetscReal low = PetscRealPart(constants[0]);
3536   const PetscReal upp = PetscRealPart(constants[1]);
3537   const PetscReal r   = PetscRealPart(u[1]);
3538   const PetscReal th  = 2. * PETSC_PI * (PetscRealPart(u[0]) - low) / (upp - low);
3539 
3540   f0[0] = r * PetscCosReal(th);
3541   f0[1] = r * PetscSinReal(th);
3542 }
3543 
3544 const char *const DMPlexShapes[] = {"box", "box_surface", "ball", "sphere", "cylinder", "schwarz_p", "gyroid", "doublet", "annulus", "unknown", "DMPlexShape", "DM_SHAPE_", NULL};
3545 
3546 static PetscErrorCode DMPlexCreateFromOptions_Internal(PetscOptionItems *PetscOptionsObject, PetscBool *useCoordSpace, DM dm)
3547 {
3548   DMPlexShape    shape   = DM_SHAPE_BOX;
3549   DMPolytopeType cell    = DM_POLYTOPE_TRIANGLE;
3550   PetscInt       dim     = 2;
3551   PetscBool      simplex = PETSC_TRUE, interpolate = PETSC_TRUE, adjCone = PETSC_FALSE, adjClosure = PETSC_TRUE, refDomain = PETSC_FALSE;
3552   PetscBool      flg, flg2, fflg, bdfflg, nameflg;
3553   MPI_Comm       comm;
3554   char           filename[PETSC_MAX_PATH_LEN]   = "<unspecified>";
3555   char           bdFilename[PETSC_MAX_PATH_LEN] = "<unspecified>";
3556   char           plexname[PETSC_MAX_PATH_LEN]   = "";
3557 
3558   PetscFunctionBegin;
3559   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3560   /* TODO Turn this into a registration interface */
3561   PetscCall(PetscOptionsString("-dm_plex_filename", "File containing a mesh", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &fflg));
3562   PetscCall(PetscOptionsString("-dm_plex_boundary_filename", "File containing a mesh boundary", "DMPlexCreateFromFile", bdFilename, bdFilename, sizeof(bdFilename), &bdfflg));
3563   PetscCall(PetscOptionsString("-dm_plex_name", "Name of the mesh in the file", "DMPlexCreateFromFile", plexname, plexname, sizeof(plexname), &nameflg));
3564   PetscCall(PetscOptionsEnum("-dm_plex_cell", "Cell shape", "", DMPolytopeTypes, (PetscEnum)cell, (PetscEnum *)&cell, NULL));
3565   PetscCall(PetscOptionsBool("-dm_plex_reference_cell_domain", "Use a reference cell domain", "", refDomain, &refDomain, NULL));
3566   PetscCall(PetscOptionsEnum("-dm_plex_shape", "Shape for built-in mesh", "", DMPlexShapes, (PetscEnum)shape, (PetscEnum *)&shape, &flg));
3567   PetscCall(PetscOptionsBoundedInt("-dm_plex_dim", "Topological dimension of the mesh", "DMGetDimension", dim, &dim, &flg, 0));
3568   PetscCheck(!(dim < 0) && !(dim > 3), comm, PETSC_ERR_ARG_OUTOFRANGE, "Dimension %" PetscInt_FMT " should be in [1, 3]", dim);
3569   PetscCall(PetscOptionsBool("-dm_plex_simplex", "Mesh cell shape", "", simplex, &simplex, &flg));
3570   PetscCall(PetscOptionsBool("-dm_plex_interpolate", "Flag to create edges and faces automatically", "", interpolate, &interpolate, &flg));
3571   PetscCall(PetscOptionsBool("-dm_plex_adj_cone", "Set adjacency direction", "DMSetBasicAdjacency", adjCone, &adjCone, &flg));
3572   PetscCall(PetscOptionsBool("-dm_plex_adj_closure", "Set adjacency size", "DMSetBasicAdjacency", adjClosure, &adjClosure, &flg2));
3573   if (flg || flg2) PetscCall(DMSetBasicAdjacency(dm, adjCone, adjClosure));
3574 
3575   switch (cell) {
3576   case DM_POLYTOPE_POINT:
3577   case DM_POLYTOPE_SEGMENT:
3578   case DM_POLYTOPE_POINT_PRISM_TENSOR:
3579   case DM_POLYTOPE_TRIANGLE:
3580   case DM_POLYTOPE_QUADRILATERAL:
3581   case DM_POLYTOPE_TETRAHEDRON:
3582   case DM_POLYTOPE_HEXAHEDRON:
3583     *useCoordSpace = PETSC_TRUE;
3584     break;
3585   default:
3586     *useCoordSpace = PETSC_FALSE;
3587     break;
3588   }
3589 
3590   if (fflg) {
3591     DM dmnew;
3592 
3593     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), filename, plexname, interpolate, &dmnew));
3594     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3595     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3596   } else if (refDomain) {
3597     PetscCall(DMPlexCreateReferenceCell_Internal(dm, cell));
3598   } else if (bdfflg) {
3599     DM bdm, dmnew;
3600 
3601     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), bdFilename, plexname, interpolate, &bdm));
3602     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)bdm, "bd_"));
3603     PetscCall(DMSetFromOptions(bdm));
3604     PetscCall(DMPlexGenerate(bdm, NULL, interpolate, &dmnew));
3605     PetscCall(DMDestroy(&bdm));
3606     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3607     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3608   } else {
3609     PetscCall(PetscObjectSetName((PetscObject)dm, DMPlexShapes[shape]));
3610     switch (shape) {
3611     case DM_SHAPE_BOX:
3612     case DM_SHAPE_ANNULUS: {
3613       PetscInt       faces[3]  = {0, 0, 0};
3614       PetscReal      lower[3]  = {0, 0, 0};
3615       PetscReal      upper[3]  = {1, 1, 1};
3616       DMBoundaryType bdt[3]    = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3617       PetscBool      isAnnular = shape == DM_SHAPE_ANNULUS ? PETSC_TRUE : PETSC_FALSE;
3618       PetscInt       i, n;
3619 
3620       n = dim;
3621       for (i = 0; i < dim; ++i) faces[i] = (dim == 1 ? 1 : 4 - dim);
3622       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3623       n = 3;
3624       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3625       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3626       n = 3;
3627       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3628       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3629       n = 3;
3630       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
3631       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3632 
3633       PetscCheck(!isAnnular || dim == 2, comm, PETSC_ERR_ARG_OUTOFRANGE, "Only two dimensional annuli have been implemented");
3634       if (isAnnular)
3635         for (i = 0; i < dim - 1; ++i) bdt[i] = DM_BOUNDARY_PERIODIC;
3636 
3637       switch (cell) {
3638       case DM_POLYTOPE_TRI_PRISM_TENSOR:
3639         PetscCall(DMPlexCreateWedgeBoxMesh_Internal(dm, faces, lower, upper, bdt));
3640         if (!interpolate) {
3641           DM udm;
3642 
3643           PetscCall(DMPlexUninterpolate(dm, &udm));
3644           PetscCall(DMPlexReplace_Internal(dm, &udm));
3645         }
3646         break;
3647       default:
3648         PetscCall(DMPlexCreateBoxMesh_Internal(dm, dim, simplex, faces, lower, upper, bdt, interpolate));
3649         break;
3650       }
3651       if (isAnnular) {
3652         DM          cdm;
3653         PetscDS     cds;
3654         PetscScalar bounds[2] = {lower[0], upper[0]};
3655 
3656         // Fix coordinates for annular region
3657         PetscCall(DMSetPeriodicity(dm, NULL, NULL, NULL));
3658         PetscCall(DMSetCellCoordinatesLocal(dm, NULL));
3659         PetscCall(DMSetCellCoordinates(dm, NULL));
3660         PetscCall(DMPlexCreateCoordinateSpace(dm, 1, NULL));
3661         PetscCall(DMGetCoordinateDM(dm, &cdm));
3662         PetscCall(DMGetDS(cdm, &cds));
3663         PetscCall(PetscDSSetConstants(cds, 2, bounds));
3664         PetscCall(DMPlexRemapGeometry(dm, 0.0, boxToAnnulus));
3665       }
3666     } break;
3667     case DM_SHAPE_BOX_SURFACE: {
3668       PetscInt  faces[3] = {0, 0, 0};
3669       PetscReal lower[3] = {0, 0, 0};
3670       PetscReal upper[3] = {1, 1, 1};
3671       PetscInt  i, n;
3672 
3673       n = dim + 1;
3674       for (i = 0; i < dim + 1; ++i) faces[i] = (dim + 1 == 1 ? 1 : 4 - (dim + 1));
3675       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3676       n = 3;
3677       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3678       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3679       n = 3;
3680       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3681       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3682       PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(dm, dim + 1, faces, lower, upper, interpolate));
3683     } break;
3684     case DM_SHAPE_SPHERE: {
3685       PetscReal R = 1.0;
3686 
3687       PetscCall(PetscOptionsReal("-dm_plex_sphere_radius", "Radius of the sphere", "", R, &R, &flg));
3688       PetscCall(DMPlexCreateSphereMesh_Internal(dm, dim, simplex, R));
3689     } break;
3690     case DM_SHAPE_BALL: {
3691       PetscReal R = 1.0;
3692 
3693       PetscCall(PetscOptionsReal("-dm_plex_ball_radius", "Radius of the ball", "", R, &R, &flg));
3694       PetscCall(DMPlexCreateBallMesh_Internal(dm, dim, R));
3695     } break;
3696     case DM_SHAPE_CYLINDER: {
3697       DMBoundaryType bdt = DM_BOUNDARY_NONE;
3698       PetscInt       Nw  = 6;
3699 
3700       PetscCall(PetscOptionsEnum("-dm_plex_cylinder_bd", "Boundary type in the z direction", "", DMBoundaryTypes, (PetscEnum)bdt, (PetscEnum *)&bdt, NULL));
3701       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_wedges", "Number of wedges around the cylinder", "", Nw, &Nw, NULL));
3702       switch (cell) {
3703       case DM_POLYTOPE_TRI_PRISM_TENSOR:
3704         PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(dm, Nw, interpolate));
3705         break;
3706       default:
3707         PetscCall(DMPlexCreateHexCylinderMesh_Internal(dm, bdt));
3708         break;
3709       }
3710     } break;
3711     case DM_SHAPE_SCHWARZ_P: // fallthrough
3712     case DM_SHAPE_GYROID: {
3713       PetscInt       extent[3] = {1, 1, 1}, refine = 0, layers = 0, three;
3714       PetscReal      thickness   = 0.;
3715       DMBoundaryType periodic[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3716       DMPlexTPSType  tps_type    = shape == DM_SHAPE_SCHWARZ_P ? DMPLEX_TPS_SCHWARZ_P : DMPLEX_TPS_GYROID;
3717       PetscBool      tps_distribute;
3718       PetscCall(PetscOptionsIntArray("-dm_plex_tps_extent", "Number of replicas for each of three dimensions", NULL, extent, (three = 3, &three), NULL));
3719       PetscCall(PetscOptionsInt("-dm_plex_tps_refine", "Number of refinements", NULL, refine, &refine, NULL));
3720       PetscCall(PetscOptionsEnumArray("-dm_plex_tps_periodic", "Periodicity in each of three dimensions", NULL, DMBoundaryTypes, (PetscEnum *)periodic, (three = 3, &three), NULL));
3721       PetscCall(PetscOptionsInt("-dm_plex_tps_layers", "Number of layers in volumetric extrusion (or zero to not extrude)", NULL, layers, &layers, NULL));
3722       PetscCall(PetscOptionsReal("-dm_plex_tps_thickness", "Thickness of volumetric extrusion", NULL, thickness, &thickness, NULL));
3723       PetscCall(DMPlexDistributeGetDefault(dm, &tps_distribute));
3724       PetscCall(PetscOptionsBool("-dm_plex_tps_distribute", "Distribute the 2D mesh prior to refinement and extrusion", NULL, tps_distribute, &tps_distribute, NULL));
3725       PetscCall(DMPlexCreateTPSMesh_Internal(dm, tps_type, extent, periodic, tps_distribute, refine, layers, thickness));
3726     } break;
3727     case DM_SHAPE_DOUBLET: {
3728       DM        dmnew;
3729       PetscReal rl = 0.0;
3730 
3731       PetscCall(PetscOptionsReal("-dm_plex_doublet_refinementlimit", "Refinement limit", NULL, rl, &rl, NULL));
3732       PetscCall(DMPlexCreateDoublet(PetscObjectComm((PetscObject)dm), dim, simplex, interpolate, rl, &dmnew));
3733       PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3734       PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3735     } break;
3736     default:
3737       SETERRQ(comm, PETSC_ERR_SUP, "Domain shape %s is unsupported", DMPlexShapes[shape]);
3738     }
3739   }
3740   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3741   if (!((PetscObject)dm)->name && nameflg) PetscCall(PetscObjectSetName((PetscObject)dm, plexname));
3742   PetscFunctionReturn(0);
3743 }
3744 
3745 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
3746 {
3747   DM_Plex  *mesh = (DM_Plex *)dm->data;
3748   PetscBool flg, flg2;
3749   char      bdLabel[PETSC_MAX_PATH_LEN];
3750 
3751   PetscFunctionBegin;
3752   /* Handle viewing */
3753   PetscCall(PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL));
3754   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fem", "Debug output level all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL, 0));
3755   PetscCall(PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL));
3756   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL, 0));
3757   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_locate", "Debug output level all point location computations", "DMLocatePoints", 0, &mesh->printLocate, NULL, 0));
3758   PetscCall(DMMonitorSetFromOptions(dm, "-dm_plex_monitor_throughput", "Monitor the simulation throughput", "DMPlexMonitorThroughput", DMPlexMonitorThroughput, NULL, &flg));
3759   if (flg) PetscCall(PetscLogDefaultBegin());
3760   /* Labeling */
3761   PetscCall(PetscOptionsString("-dm_plex_boundary_label", "Label to mark the mesh boundary", "", bdLabel, bdLabel, sizeof(bdLabel), &flg));
3762   if (flg) PetscCall(DMPlexCreateBoundaryLabel_Private(dm, bdLabel));
3763   /* Point Location */
3764   PetscCall(PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL));
3765   /* Partitioning and distribution */
3766   PetscCall(PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL));
3767   /* Generation and remeshing */
3768   PetscCall(PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL));
3769   /* Projection behavior */
3770   PetscCall(PetscOptionsBoundedInt("-dm_plex_max_projection_height", "Maximum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL, 0));
3771   PetscCall(PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL));
3772   /* Checking structure */
3773   {
3774     PetscBool all = PETSC_FALSE;
3775 
3776     PetscCall(PetscOptionsBool("-dm_plex_check_all", "Perform all basic checks", "DMPlexCheck", PETSC_FALSE, &all, NULL));
3777     if (all) {
3778       PetscCall(DMPlexCheck(dm));
3779     } else {
3780       PetscCall(PetscOptionsBool("-dm_plex_check_symmetry", "Check that the adjacency information in the mesh is symmetric", "DMPlexCheckSymmetry", PETSC_FALSE, &flg, &flg2));
3781       if (flg && flg2) PetscCall(DMPlexCheckSymmetry(dm));
3782       PetscCall(PetscOptionsBool("-dm_plex_check_skeleton", "Check that each cell has the correct number of vertices (only for homogeneous simplex or tensor meshes)", "DMPlexCheckSkeleton", PETSC_FALSE, &flg, &flg2));
3783       if (flg && flg2) PetscCall(DMPlexCheckSkeleton(dm, 0));
3784       PetscCall(PetscOptionsBool("-dm_plex_check_faces", "Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type", "DMPlexCheckFaces", PETSC_FALSE, &flg, &flg2));
3785       if (flg && flg2) PetscCall(DMPlexCheckFaces(dm, 0));
3786       PetscCall(PetscOptionsBool("-dm_plex_check_geometry", "Check that cells have positive volume", "DMPlexCheckGeometry", PETSC_FALSE, &flg, &flg2));
3787       if (flg && flg2) PetscCall(DMPlexCheckGeometry(dm));
3788       PetscCall(PetscOptionsBool("-dm_plex_check_pointsf", "Check some necessary conditions for PointSF", "DMPlexCheckPointSF", PETSC_FALSE, &flg, &flg2));
3789       if (flg && flg2) PetscCall(DMPlexCheckPointSF(dm, NULL, PETSC_FALSE));
3790       PetscCall(PetscOptionsBool("-dm_plex_check_interface_cones", "Check points on inter-partition interfaces have conforming order of cone points", "DMPlexCheckInterfaceCones", PETSC_FALSE, &flg, &flg2));
3791       if (flg && flg2) PetscCall(DMPlexCheckInterfaceCones(dm));
3792     }
3793     PetscCall(PetscOptionsBool("-dm_plex_check_cell_shape", "Check cell shape", "DMPlexCheckCellShape", PETSC_FALSE, &flg, &flg2));
3794     if (flg && flg2) PetscCall(DMPlexCheckCellShape(dm, PETSC_TRUE, PETSC_DETERMINE));
3795   }
3796   {
3797     PetscReal scale = 1.0;
3798 
3799     PetscCall(PetscOptionsReal("-dm_plex_scale", "Scale factor for mesh coordinates", "DMPlexScale", scale, &scale, &flg));
3800     if (flg) {
3801       Vec coordinates, coordinatesLocal;
3802 
3803       PetscCall(DMGetCoordinates(dm, &coordinates));
3804       PetscCall(DMGetCoordinatesLocal(dm, &coordinatesLocal));
3805       PetscCall(VecScale(coordinates, scale));
3806       PetscCall(VecScale(coordinatesLocal, scale));
3807     }
3808   }
3809   PetscCall(PetscPartitionerSetFromOptions(mesh->partitioner));
3810   PetscFunctionReturn(0);
3811 }
3812 
3813 PetscErrorCode DMSetFromOptions_Overlap_Plex(DM dm, PetscOptionItems *PetscOptionsObject, PetscInt *overlap)
3814 {
3815   PetscInt  numOvLabels = 16, numOvExLabels = 16;
3816   char     *ovLabelNames[16], *ovExLabelNames[16];
3817   PetscInt  numOvValues = 16, numOvExValues = 16, l;
3818   PetscBool flg;
3819 
3820   PetscFunctionBegin;
3821   PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", *overlap, overlap, NULL, 0));
3822   PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_labels", "List of overlap label names", "DMPlexDistribute", ovLabelNames, &numOvLabels, &flg));
3823   if (!flg) numOvLabels = 0;
3824   if (numOvLabels) {
3825     ((DM_Plex *)dm->data)->numOvLabels = numOvLabels;
3826     for (l = 0; l < numOvLabels; ++l) {
3827       PetscCall(DMGetLabel(dm, ovLabelNames[l], &((DM_Plex *)dm->data)->ovLabels[l]));
3828       PetscCheck(((DM_Plex *)dm->data)->ovLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovLabelNames[l]);
3829       PetscCall(PetscFree(ovLabelNames[l]));
3830     }
3831     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_values", "List of overlap label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovValues, &numOvValues, &flg));
3832     if (!flg) numOvValues = 0;
3833     PetscCheck(numOvLabels == numOvValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvLabels, numOvValues);
3834 
3835     PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_exclude_labels", "List of overlap exclude label names", "DMPlexDistribute", ovExLabelNames, &numOvExLabels, &flg));
3836     if (!flg) numOvExLabels = 0;
3837     ((DM_Plex *)dm->data)->numOvExLabels = numOvExLabels;
3838     for (l = 0; l < numOvExLabels; ++l) {
3839       PetscCall(DMGetLabel(dm, ovExLabelNames[l], &((DM_Plex *)dm->data)->ovExLabels[l]));
3840       PetscCheck(((DM_Plex *)dm->data)->ovExLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovExLabelNames[l]);
3841       PetscCall(PetscFree(ovExLabelNames[l]));
3842     }
3843     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_exclude_values", "List of overlap exclude label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovExValues, &numOvExValues, &flg));
3844     if (!flg) numOvExValues = 0;
3845     PetscCheck(numOvExLabels == numOvExValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of exclude labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvExLabels, numOvExValues);
3846   }
3847   PetscFunctionReturn(0);
3848 }
3849 
3850 static PetscErrorCode DMSetFromOptions_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
3851 {
3852   PetscFunctionList        ordlist;
3853   char                     oname[256];
3854   PetscReal                volume    = -1.0;
3855   PetscInt                 prerefine = 0, refine = 0, r, coarsen = 0, overlap = 0, extLayers = 0, dim;
3856   PetscBool                uniformOrig, created = PETSC_FALSE, uniform = PETSC_TRUE, distribute, interpolate = PETSC_TRUE, coordSpace = PETSC_TRUE, remap = PETSC_TRUE, ghostCells = PETSC_FALSE, isHierarchy, ignoreModel = PETSC_FALSE, flg;
3857   DMPlexReorderDefaultFlag reorder;
3858 
3859   PetscFunctionBegin;
3860   PetscOptionsHeadBegin(PetscOptionsObject, "DMPlex Options");
3861   if (dm->cloneOpts) goto non_refine;
3862   /* Handle automatic creation */
3863   PetscCall(DMGetDimension(dm, &dim));
3864   if (dim < 0) {
3865     PetscCall(DMPlexCreateFromOptions_Internal(PetscOptionsObject, &coordSpace, dm));
3866     created = PETSC_TRUE;
3867   }
3868   PetscCall(DMGetDimension(dm, &dim));
3869   /* Handle interpolation before distribution */
3870   PetscCall(PetscOptionsBool("-dm_plex_interpolate_pre", "Flag to interpolate mesh before distribution", "", interpolate, &interpolate, &flg));
3871   if (flg) {
3872     DMPlexInterpolatedFlag interpolated;
3873 
3874     PetscCall(DMPlexIsInterpolated(dm, &interpolated));
3875     if (interpolated == DMPLEX_INTERPOLATED_FULL && !interpolate) {
3876       DM udm;
3877 
3878       PetscCall(DMPlexUninterpolate(dm, &udm));
3879       PetscCall(DMPlexReplace_Internal(dm, &udm));
3880     } else if (interpolated != DMPLEX_INTERPOLATED_FULL && interpolate) {
3881       DM idm;
3882 
3883       PetscCall(DMPlexInterpolate(dm, &idm));
3884       PetscCall(DMPlexReplace_Internal(dm, &idm));
3885     }
3886   }
3887   /* Handle DMPlex refinement before distribution */
3888   PetscCall(PetscOptionsBool("-dm_refine_ignore_model", "Flag to ignore the geometry model when refining", "DMCreate", ignoreModel, &ignoreModel, &flg));
3889   if (flg) ((DM_Plex *)dm->data)->ignoreModel = ignoreModel;
3890   PetscCall(DMPlexGetRefinementUniform(dm, &uniformOrig));
3891   PetscCall(PetscOptionsBoundedInt("-dm_refine_pre", "The number of refinements before distribution", "DMCreate", prerefine, &prerefine, NULL, 0));
3892   PetscCall(PetscOptionsBool("-dm_refine_remap_pre", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
3893   PetscCall(PetscOptionsBool("-dm_refine_uniform_pre", "Flag for uniform refinement before distribution", "DMCreate", uniform, &uniform, &flg));
3894   if (flg) PetscCall(DMPlexSetRefinementUniform(dm, uniform));
3895   PetscCall(PetscOptionsReal("-dm_refine_volume_limit_pre", "The maximum cell volume after refinement before distribution", "DMCreate", volume, &volume, &flg));
3896   if (flg) {
3897     PetscCall(DMPlexSetRefinementUniform(dm, PETSC_FALSE));
3898     PetscCall(DMPlexSetRefinementLimit(dm, volume));
3899     prerefine = PetscMax(prerefine, 1);
3900   }
3901   for (r = 0; r < prerefine; ++r) {
3902     DM             rdm;
3903     PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
3904 
3905     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3906     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
3907     PetscCall(DMPlexReplace_Internal(dm, &rdm));
3908     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3909     if (coordFunc && remap) {
3910       PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
3911       ((DM_Plex *)dm->data)->coordFunc = coordFunc;
3912     }
3913   }
3914   PetscCall(DMPlexSetRefinementUniform(dm, uniformOrig));
3915   /* Handle DMPlex extrusion before distribution */
3916   PetscCall(PetscOptionsBoundedInt("-dm_extrude", "The number of layers to extrude", "", extLayers, &extLayers, NULL, 0));
3917   if (extLayers) {
3918     DM edm;
3919 
3920     PetscCall(DMExtrude(dm, extLayers, &edm));
3921     PetscCall(DMPlexReplace_Internal(dm, &edm));
3922     ((DM_Plex *)dm->data)->coordFunc = NULL;
3923     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3924     extLayers = 0;
3925     PetscCall(DMGetDimension(dm, &dim));
3926   }
3927   /* Handle DMPlex reordering before distribution */
3928   PetscCall(DMPlexReorderGetDefault(dm, &reorder));
3929   PetscCall(MatGetOrderingList(&ordlist));
3930   PetscCall(PetscStrncpy(oname, MATORDERINGNATURAL, sizeof(oname)));
3931   PetscCall(PetscOptionsFList("-dm_plex_reorder", "Set mesh reordering type", "DMPlexGetOrdering", ordlist, MATORDERINGNATURAL, oname, sizeof(oname), &flg));
3932   if (reorder == DMPLEX_REORDER_DEFAULT_TRUE || flg) {
3933     DM pdm;
3934     IS perm;
3935 
3936     PetscCall(DMPlexGetOrdering(dm, oname, NULL, &perm));
3937     PetscCall(DMPlexPermute(dm, perm, &pdm));
3938     PetscCall(ISDestroy(&perm));
3939     PetscCall(DMPlexReplace_Internal(dm, &pdm));
3940     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3941   }
3942   /* Handle DMPlex distribution */
3943   PetscCall(DMPlexDistributeGetDefault(dm, &distribute));
3944   PetscCall(PetscOptionsBool("-dm_distribute", "Flag to redistribute a mesh among processes", "DMPlexDistribute", distribute, &distribute, NULL));
3945   PetscCall(DMSetFromOptions_Overlap_Plex(dm, PetscOptionsObject, &overlap));
3946   if (distribute) {
3947     DM               pdm = NULL;
3948     PetscPartitioner part;
3949 
3950     PetscCall(DMPlexGetPartitioner(dm, &part));
3951     PetscCall(PetscPartitionerSetFromOptions(part));
3952     PetscCall(DMPlexDistribute(dm, overlap, NULL, &pdm));
3953     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
3954   }
3955   /* Create coordinate space */
3956   if (created) {
3957     DM_Plex  *mesh   = (DM_Plex *)dm->data;
3958     PetscInt  degree = 1;
3959     PetscBool flg;
3960 
3961     PetscCall(PetscOptionsBool("-dm_coord_space", "Use an FEM space for coordinates", "", coordSpace, &coordSpace, &flg));
3962     PetscCall(PetscOptionsInt("-dm_coord_petscspace_degree", "FEM degree for coordinate space", "", degree, &degree, NULL));
3963     if (coordSpace) PetscCall(DMPlexCreateCoordinateSpace(dm, degree, mesh->coordFunc));
3964     if (flg && !coordSpace) {
3965       DM           cdm;
3966       PetscDS      cds;
3967       PetscObject  obj;
3968       PetscClassId id;
3969 
3970       PetscCall(DMGetCoordinateDM(dm, &cdm));
3971       PetscCall(DMGetDS(cdm, &cds));
3972       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
3973       PetscCall(PetscObjectGetClassId(obj, &id));
3974       if (id == PETSCFE_CLASSID) {
3975         PetscContainer dummy;
3976 
3977         PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &dummy));
3978         PetscCall(PetscObjectSetName((PetscObject)dummy, "coordinates"));
3979         PetscCall(DMSetField(cdm, 0, NULL, (PetscObject)dummy));
3980         PetscCall(PetscContainerDestroy(&dummy));
3981         PetscCall(DMClearDS(cdm));
3982       }
3983       mesh->coordFunc = NULL;
3984     }
3985     PetscCall(PetscOptionsBool("-dm_sparse_localize", "Localize only necessary cells", "", dm->sparseLocalize, &dm->sparseLocalize, &flg));
3986     PetscCall(DMLocalizeCoordinates(dm));
3987   }
3988   /* Handle DMPlex refinement */
3989   remap = PETSC_TRUE;
3990   PetscCall(PetscOptionsBoundedInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL, 0));
3991   PetscCall(PetscOptionsBool("-dm_refine_remap", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
3992   PetscCall(PetscOptionsBoundedInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy, 0));
3993   if (refine) PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3994   if (refine && isHierarchy) {
3995     DM *dms, coarseDM;
3996 
3997     PetscCall(DMGetCoarseDM(dm, &coarseDM));
3998     PetscCall(PetscObjectReference((PetscObject)coarseDM));
3999     PetscCall(PetscMalloc1(refine, &dms));
4000     PetscCall(DMRefineHierarchy(dm, refine, dms));
4001     /* Total hack since we do not pass in a pointer */
4002     PetscCall(DMPlexSwap_Static(dm, dms[refine - 1]));
4003     if (refine == 1) {
4004       PetscCall(DMSetCoarseDM(dm, dms[0]));
4005       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4006     } else {
4007       PetscCall(DMSetCoarseDM(dm, dms[refine - 2]));
4008       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4009       PetscCall(DMSetCoarseDM(dms[0], dms[refine - 1]));
4010       PetscCall(DMPlexSetRegularRefinement(dms[0], PETSC_TRUE));
4011     }
4012     PetscCall(DMSetCoarseDM(dms[refine - 1], coarseDM));
4013     PetscCall(PetscObjectDereference((PetscObject)coarseDM));
4014     /* Free DMs */
4015     for (r = 0; r < refine; ++r) {
4016       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4017       PetscCall(DMDestroy(&dms[r]));
4018     }
4019     PetscCall(PetscFree(dms));
4020   } else {
4021     for (r = 0; r < refine; ++r) {
4022       DM             rdm;
4023       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4024 
4025       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4026       PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
4027       /* Total hack since we do not pass in a pointer */
4028       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4029       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4030       if (coordFunc && remap) {
4031         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4032         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4033       }
4034     }
4035   }
4036   /* Handle DMPlex coarsening */
4037   PetscCall(PetscOptionsBoundedInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL, 0));
4038   PetscCall(PetscOptionsBoundedInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy, 0));
4039   if (coarsen && isHierarchy) {
4040     DM *dms;
4041 
4042     PetscCall(PetscMalloc1(coarsen, &dms));
4043     PetscCall(DMCoarsenHierarchy(dm, coarsen, dms));
4044     /* Free DMs */
4045     for (r = 0; r < coarsen; ++r) {
4046       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4047       PetscCall(DMDestroy(&dms[r]));
4048     }
4049     PetscCall(PetscFree(dms));
4050   } else {
4051     for (r = 0; r < coarsen; ++r) {
4052       DM             cdm;
4053       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4054 
4055       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4056       PetscCall(DMCoarsen(dm, PetscObjectComm((PetscObject)dm), &cdm));
4057       /* Total hack since we do not pass in a pointer */
4058       PetscCall(DMPlexReplace_Internal(dm, &cdm));
4059       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4060       if (coordFunc) {
4061         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4062         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4063       }
4064     }
4065   }
4066   /* Handle ghost cells */
4067   PetscCall(PetscOptionsBool("-dm_plex_create_fv_ghost_cells", "Flag to create finite volume ghost cells on the boundary", "DMCreate", ghostCells, &ghostCells, NULL));
4068   if (ghostCells) {
4069     DM   gdm;
4070     char lname[PETSC_MAX_PATH_LEN];
4071 
4072     lname[0] = '\0';
4073     PetscCall(PetscOptionsString("-dm_plex_fv_ghost_cells_label", "Label name for ghost cells boundary", "DMCreate", lname, lname, sizeof(lname), &flg));
4074     PetscCall(DMPlexConstructGhostCells(dm, flg ? lname : NULL, NULL, &gdm));
4075     PetscCall(DMPlexReplace_Internal(dm, &gdm));
4076   }
4077   /* Handle 1D order */
4078   if (reorder != DMPLEX_REORDER_DEFAULT_FALSE && dim == 1) {
4079     DM           cdm, rdm;
4080     PetscDS      cds;
4081     PetscObject  obj;
4082     PetscClassId id = PETSC_OBJECT_CLASSID;
4083     IS           perm;
4084     PetscInt     Nf;
4085     PetscBool    distributed;
4086 
4087     PetscCall(DMPlexIsDistributed(dm, &distributed));
4088     PetscCall(DMGetCoordinateDM(dm, &cdm));
4089     PetscCall(DMGetDS(cdm, &cds));
4090     PetscCall(PetscDSGetNumFields(cds, &Nf));
4091     if (Nf) {
4092       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
4093       PetscCall(PetscObjectGetClassId(obj, &id));
4094     }
4095     if (!distributed && id != PETSCFE_CLASSID) {
4096       PetscCall(DMPlexGetOrdering1D(dm, &perm));
4097       PetscCall(DMPlexPermute(dm, perm, &rdm));
4098       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4099       PetscCall(ISDestroy(&perm));
4100     }
4101   }
4102 /* Handle */
4103 non_refine:
4104   PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4105   PetscOptionsHeadEnd();
4106   PetscFunctionReturn(0);
4107 }
4108 
4109 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm, Vec *vec)
4110 {
4111   PetscFunctionBegin;
4112   PetscCall(DMCreateGlobalVector_Section_Private(dm, vec));
4113   /* PetscCall(VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM)); */
4114   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex));
4115   PetscCall(VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void))VecView_Plex_Native));
4116   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex));
4117   PetscCall(VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void))VecLoad_Plex_Native));
4118   PetscFunctionReturn(0);
4119 }
4120 
4121 static PetscErrorCode DMCreateLocalVector_Plex(DM dm, Vec *vec)
4122 {
4123   PetscFunctionBegin;
4124   PetscCall(DMCreateLocalVector_Section_Private(dm, vec));
4125   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex_Local));
4126   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex_Local));
4127   PetscFunctionReturn(0);
4128 }
4129 
4130 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
4131 {
4132   PetscInt depth, d;
4133 
4134   PetscFunctionBegin;
4135   PetscCall(DMPlexGetDepth(dm, &depth));
4136   if (depth == 1) {
4137     PetscCall(DMGetDimension(dm, &d));
4138     if (dim == 0) PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4139     else if (dim == d) PetscCall(DMPlexGetDepthStratum(dm, 1, pStart, pEnd));
4140     else {
4141       *pStart = 0;
4142       *pEnd   = 0;
4143     }
4144   } else {
4145     PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4146   }
4147   PetscFunctionReturn(0);
4148 }
4149 
4150 static PetscErrorCode DMGetNeighbors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
4151 {
4152   PetscSF            sf;
4153   PetscInt           niranks, njranks, n;
4154   const PetscMPIInt *iranks, *jranks;
4155   DM_Plex           *data = (DM_Plex *)dm->data;
4156 
4157   PetscFunctionBegin;
4158   PetscCall(DMGetPointSF(dm, &sf));
4159   if (!data->neighbors) {
4160     PetscCall(PetscSFSetUp(sf));
4161     PetscCall(PetscSFGetRootRanks(sf, &njranks, &jranks, NULL, NULL, NULL));
4162     PetscCall(PetscSFGetLeafRanks(sf, &niranks, &iranks, NULL, NULL));
4163     PetscCall(PetscMalloc1(njranks + niranks + 1, &data->neighbors));
4164     PetscCall(PetscArraycpy(data->neighbors + 1, jranks, njranks));
4165     PetscCall(PetscArraycpy(data->neighbors + njranks + 1, iranks, niranks));
4166     n = njranks + niranks;
4167     PetscCall(PetscSortRemoveDupsMPIInt(&n, data->neighbors + 1));
4168     /* The following cast should never fail: can't have more neighbors than PETSC_MPI_INT_MAX */
4169     PetscCall(PetscMPIIntCast(n, data->neighbors));
4170   }
4171   if (nranks) *nranks = data->neighbors[0];
4172   if (ranks) {
4173     if (data->neighbors[0]) *ranks = data->neighbors + 1;
4174     else *ranks = NULL;
4175   }
4176   PetscFunctionReturn(0);
4177 }
4178 
4179 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM, DM, Mat, Vec, Vec);
4180 
4181 static PetscErrorCode DMInitialize_Plex(DM dm)
4182 {
4183   PetscFunctionBegin;
4184   dm->ops->view                      = DMView_Plex;
4185   dm->ops->load                      = DMLoad_Plex;
4186   dm->ops->setfromoptions            = DMSetFromOptions_Plex;
4187   dm->ops->clone                     = DMClone_Plex;
4188   dm->ops->setup                     = DMSetUp_Plex;
4189   dm->ops->createlocalsection        = DMCreateLocalSection_Plex;
4190   dm->ops->createdefaultconstraints  = DMCreateDefaultConstraints_Plex;
4191   dm->ops->createglobalvector        = DMCreateGlobalVector_Plex;
4192   dm->ops->createlocalvector         = DMCreateLocalVector_Plex;
4193   dm->ops->getlocaltoglobalmapping   = NULL;
4194   dm->ops->createfieldis             = NULL;
4195   dm->ops->createcoordinatedm        = DMCreateCoordinateDM_Plex;
4196   dm->ops->createcoordinatefield     = DMCreateCoordinateField_Plex;
4197   dm->ops->getcoloring               = NULL;
4198   dm->ops->creatematrix              = DMCreateMatrix_Plex;
4199   dm->ops->createinterpolation       = DMCreateInterpolation_Plex;
4200   dm->ops->createmassmatrix          = DMCreateMassMatrix_Plex;
4201   dm->ops->createmassmatrixlumped    = DMCreateMassMatrixLumped_Plex;
4202   dm->ops->createinjection           = DMCreateInjection_Plex;
4203   dm->ops->refine                    = DMRefine_Plex;
4204   dm->ops->coarsen                   = DMCoarsen_Plex;
4205   dm->ops->refinehierarchy           = DMRefineHierarchy_Plex;
4206   dm->ops->coarsenhierarchy          = DMCoarsenHierarchy_Plex;
4207   dm->ops->extrude                   = DMExtrude_Plex;
4208   dm->ops->globaltolocalbegin        = NULL;
4209   dm->ops->globaltolocalend          = NULL;
4210   dm->ops->localtoglobalbegin        = NULL;
4211   dm->ops->localtoglobalend          = NULL;
4212   dm->ops->destroy                   = DMDestroy_Plex;
4213   dm->ops->createsubdm               = DMCreateSubDM_Plex;
4214   dm->ops->createsuperdm             = DMCreateSuperDM_Plex;
4215   dm->ops->getdimpoints              = DMGetDimPoints_Plex;
4216   dm->ops->locatepoints              = DMLocatePoints_Plex;
4217   dm->ops->projectfunctionlocal      = DMProjectFunctionLocal_Plex;
4218   dm->ops->projectfunctionlabellocal = DMProjectFunctionLabelLocal_Plex;
4219   dm->ops->projectfieldlocal         = DMProjectFieldLocal_Plex;
4220   dm->ops->projectfieldlabellocal    = DMProjectFieldLabelLocal_Plex;
4221   dm->ops->projectbdfieldlabellocal  = DMProjectBdFieldLabelLocal_Plex;
4222   dm->ops->computel2diff             = DMComputeL2Diff_Plex;
4223   dm->ops->computel2gradientdiff     = DMComputeL2GradientDiff_Plex;
4224   dm->ops->computel2fielddiff        = DMComputeL2FieldDiff_Plex;
4225   dm->ops->getneighbors              = DMGetNeighbors_Plex;
4226   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertBoundaryValues_C", DMPlexInsertBoundaryValues_Plex));
4227   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertTimeDerviativeBoundaryValues_C", DMPlexInsertTimeDerivativeBoundaryValues_Plex));
4228   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMSetUpGLVisViewer_C", DMSetUpGLVisViewer_Plex));
4229   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMCreateNeumannOverlap_C", DMCreateNeumannOverlap_Plex));
4230   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4231   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeGetDefault_C", DMPlexDistributeGetDefault_Plex));
4232   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeSetDefault_C", DMPlexDistributeSetDefault_Plex));
4233   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderGetDefault_C", DMPlexReorderGetDefault_Plex));
4234   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderSetDefault_C", DMPlexReorderSetDefault_Plex));
4235   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMInterpolateSolution_C", DMInterpolateSolution_Plex));
4236   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4237   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetOverlap_C", DMPlexSetOverlap_Plex));
4238   PetscFunctionReturn(0);
4239 }
4240 
4241 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
4242 {
4243   DM_Plex *mesh = (DM_Plex *)dm->data;
4244 
4245   PetscFunctionBegin;
4246   mesh->refct++;
4247   (*newdm)->data = mesh;
4248   PetscCall(PetscObjectChangeTypeName((PetscObject)*newdm, DMPLEX));
4249   PetscCall(DMInitialize_Plex(*newdm));
4250   PetscFunctionReturn(0);
4251 }
4252 
4253 /*MC
4254   DMPLEX = "plex" - A DM object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
4255                     In the local representation, Vecs contain all unknowns in the interior and shared boundary. This is
4256                     specified by a PetscSection object. Ownership in the global representation is determined by
4257                     ownership of the underlying DMPlex points. This is specified by another PetscSection object.
4258 
4259   Options Database Keys:
4260 + -dm_refine_pre                     - Refine mesh before distribution
4261 + -dm_refine_uniform_pre             - Choose uniform or generator-based refinement
4262 + -dm_refine_volume_limit_pre        - Cell volume limit after pre-refinement using generator
4263 . -dm_distribute                     - Distribute mesh across processes
4264 . -dm_distribute_overlap             - Number of cells to overlap for distribution
4265 . -dm_refine                         - Refine mesh after distribution
4266 . -dm_plex_hash_location             - Use grid hashing for point location
4267 . -dm_plex_hash_box_faces <n,m,p>    - The number of divisions in each direction of the grid hash
4268 . -dm_plex_partition_balance         - Attempt to evenly divide points on partition boundary between processes
4269 . -dm_plex_remesh_bd                 - Allow changes to the boundary on remeshing
4270 . -dm_plex_max_projection_height     - Maximum mesh point height used to project locally
4271 . -dm_plex_regular_refinement        - Use special nested projection algorithm for regular refinement
4272 . -dm_plex_check_all                 - Perform all shecks below
4273 . -dm_plex_check_symmetry            - Check that the adjacency information in the mesh is symmetric
4274 . -dm_plex_check_skeleton <celltype> - Check that each cell has the correct number of vertices
4275 . -dm_plex_check_faces <celltype>    - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
4276 . -dm_plex_check_geometry            - Check that cells have positive volume
4277 . -dm_view :mesh.tex:ascii_latex     - View the mesh in LaTeX/TikZ
4278 . -dm_plex_view_scale <num>          - Scale the TikZ
4279 - -dm_plex_print_fem <num>           - View FEM assembly information, such as element vectors and matrices
4280 
4281   Level: intermediate
4282 
4283 .seealso: `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`
4284 M*/
4285 
4286 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
4287 {
4288   DM_Plex *mesh;
4289   PetscInt unit;
4290 
4291   PetscFunctionBegin;
4292   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4293   PetscCall(PetscNew(&mesh));
4294   dm->data = mesh;
4295 
4296   mesh->refct = 1;
4297   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection));
4298   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection));
4299   mesh->refinementUniform      = PETSC_TRUE;
4300   mesh->refinementLimit        = -1.0;
4301   mesh->distDefault            = PETSC_TRUE;
4302   mesh->reorderDefault         = DMPLEX_REORDER_DEFAULT_NOTSET;
4303   mesh->distributionName       = NULL;
4304   mesh->interpolated           = DMPLEX_INTERPOLATED_INVALID;
4305   mesh->interpolatedCollective = DMPLEX_INTERPOLATED_INVALID;
4306 
4307   PetscCall(PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner));
4308   mesh->remeshBd = PETSC_FALSE;
4309 
4310   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
4311 
4312   mesh->depthState    = -1;
4313   mesh->celltypeState = -1;
4314   mesh->printTol      = 1.0e-10;
4315 
4316   PetscCall(DMInitialize_Plex(dm));
4317   PetscFunctionReturn(0);
4318 }
4319 
4320 /*@
4321   DMPlexCreate - Creates a DMPlex object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
4322 
4323   Collective
4324 
4325   Input Parameter:
4326 . comm - The communicator for the DMPlex object
4327 
4328   Output Parameter:
4329 . mesh  - The DMPlex object
4330 
4331   Level: beginner
4332 
4333 @*/
4334 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
4335 {
4336   PetscFunctionBegin;
4337   PetscValidPointer(mesh, 2);
4338   PetscCall(DMCreate(comm, mesh));
4339   PetscCall(DMSetType(*mesh, DMPLEX));
4340   PetscFunctionReturn(0);
4341 }
4342 
4343 /*@C
4344   DMPlexBuildFromCellListParallel - Build distributed DMPLEX topology from a list of vertices for each cell (common mesh generator output)
4345 
4346   Input Parameters:
4347 + dm - The DM
4348 . numCells - The number of cells owned by this process
4349 . numVertices - The number of vertices to be owned by this process, or PETSC_DECIDE
4350 . NVertices - The global number of vertices, or PETSC_DETERMINE
4351 . numCorners - The number of vertices for each cell
4352 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4353 
4354   Output Parameters:
4355 + vertexSF - (Optional) SF describing complete vertex ownership
4356 - verticesAdjSaved - (Optional) vertex adjacency array
4357 
4358   Notes:
4359   Two triangles sharing a face
4360 $
4361 $        2
4362 $      / | \
4363 $     /  |  \
4364 $    /   |   \
4365 $   0  0 | 1  3
4366 $    \   |   /
4367 $     \  |  /
4368 $      \ | /
4369 $        1
4370 would have input
4371 $  numCells = 2, numVertices = 4
4372 $  cells = [0 1 2  1 3 2]
4373 $
4374 which would result in the DMPlex
4375 $
4376 $        4
4377 $      / | \
4378 $     /  |  \
4379 $    /   |   \
4380 $   2  0 | 1  5
4381 $    \   |   /
4382 $     \  |  /
4383 $      \ | /
4384 $        3
4385 
4386   Vertices are implicitly numbered consecutively 0,...,NVertices.
4387   Each rank owns a chunk of numVertices consecutive vertices.
4388   If numVertices is PETSC_DECIDE, PETSc will distribute them as evenly as possible using PetscLayout.
4389   If NVertices is PETSC_DETERMINE and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
4390   If only NVertices is PETSC_DETERMINE, it is computed as the sum of numVertices over all ranks.
4391 
4392   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
4393 
4394   Not currently supported in Fortran.
4395 
4396   Level: advanced
4397 
4398 .seealso: `DMPlexBuildFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildCoordinatesFromCellListParallel()`
4399 @*/
4400 PetscErrorCode DMPlexBuildFromCellListParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, const PetscInt cells[], PetscSF *vertexSF, PetscInt **verticesAdjSaved)
4401 {
4402   PetscSF     sfPoint;
4403   PetscLayout layout;
4404   PetscInt    numVerticesAdj, *verticesAdj, *cones, c, p;
4405 
4406   PetscFunctionBegin;
4407   PetscValidLogicalCollectiveInt(dm, NVertices, 4);
4408   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4409   /* Get/check global number of vertices */
4410   {
4411     PetscInt       NVerticesInCells, i;
4412     const PetscInt len = numCells * numCorners;
4413 
4414     /* NVerticesInCells = max(cells) + 1 */
4415     NVerticesInCells = PETSC_MIN_INT;
4416     for (i = 0; i < len; i++)
4417       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4418     ++NVerticesInCells;
4419     PetscCallMPI(MPI_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
4420 
4421     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
4422     else
4423       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
4424   }
4425   /* Count locally unique vertices */
4426   {
4427     PetscHSetI vhash;
4428     PetscInt   off = 0;
4429 
4430     PetscCall(PetscHSetICreate(&vhash));
4431     for (c = 0; c < numCells; ++c) {
4432       for (p = 0; p < numCorners; ++p) PetscCall(PetscHSetIAdd(vhash, cells[c * numCorners + p]));
4433     }
4434     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
4435     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
4436     else verticesAdj = *verticesAdjSaved;
4437     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
4438     PetscCall(PetscHSetIDestroy(&vhash));
4439     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
4440   }
4441   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
4442   /* Create cones */
4443   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
4444   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
4445   PetscCall(DMSetUp(dm));
4446   PetscCall(DMPlexGetCones(dm, &cones));
4447   for (c = 0; c < numCells; ++c) {
4448     for (p = 0; p < numCorners; ++p) {
4449       const PetscInt gv = cells[c * numCorners + p];
4450       PetscInt       lv;
4451 
4452       /* Positions within verticesAdj form 0-based local vertex numbering;
4453          we need to shift it by numCells to get correct DAG points (cells go first) */
4454       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
4455       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
4456       cones[c * numCorners + p] = lv + numCells;
4457     }
4458   }
4459   /* Build point sf */
4460   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
4461   PetscCall(PetscLayoutSetSize(layout, NVertices));
4462   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
4463   PetscCall(PetscLayoutSetBlockSize(layout, 1));
4464   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
4465   PetscCall(PetscLayoutDestroy(&layout));
4466   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
4467   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
4468   if (dm->sf) {
4469     const char *prefix;
4470 
4471     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
4472     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
4473   }
4474   PetscCall(DMSetPointSF(dm, sfPoint));
4475   PetscCall(PetscSFDestroy(&sfPoint));
4476   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)(*vertexSF), "Vertex Ownership SF"));
4477   /* Fill in the rest of the topology structure */
4478   PetscCall(DMPlexSymmetrize(dm));
4479   PetscCall(DMPlexStratify(dm));
4480   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4481   PetscFunctionReturn(0);
4482 }
4483 
4484 /*@C
4485   DMPlexBuildCoordinatesFromCellListParallel - Build DM coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4486 
4487   Input Parameters:
4488 + dm - The DM
4489 . spaceDim - The spatial dimension used for coordinates
4490 . sfVert - SF describing complete vertex ownership
4491 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4492 
4493   Level: advanced
4494 
4495   Notes:
4496   Not currently supported in Fortran.
4497 
4498 .seealso: `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellListParallel()`
4499 @*/
4500 PetscErrorCode DMPlexBuildCoordinatesFromCellListParallel(DM dm, PetscInt spaceDim, PetscSF sfVert, const PetscReal vertexCoords[])
4501 {
4502   PetscSection coordSection;
4503   Vec          coordinates;
4504   PetscScalar *coords;
4505   PetscInt     numVertices, numVerticesAdj, coordSize, v, vStart, vEnd;
4506 
4507   PetscFunctionBegin;
4508   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4509   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
4510   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4511   PetscCall(DMSetCoordinateDim(dm, spaceDim));
4512   PetscCall(PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL));
4513   PetscCheck(vEnd - vStart == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Supplied sfVert has wrong number of leaves = %" PetscInt_FMT " != %" PetscInt_FMT " = vEnd - vStart", numVerticesAdj, vEnd - vStart);
4514   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4515   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4516   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
4517   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
4518   for (v = vStart; v < vEnd; ++v) {
4519     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
4520     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
4521   }
4522   PetscCall(PetscSectionSetUp(coordSection));
4523   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4524   PetscCall(VecCreate(PetscObjectComm((PetscObject)dm), &coordinates));
4525   PetscCall(VecSetBlockSize(coordinates, spaceDim));
4526   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4527   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4528   PetscCall(VecSetType(coordinates, VECSTANDARD));
4529   PetscCall(VecGetArray(coordinates, &coords));
4530   {
4531     MPI_Datatype coordtype;
4532 
4533     /* Need a temp buffer for coords if we have complex/single */
4534     PetscCallMPI(MPI_Type_contiguous(spaceDim, MPIU_SCALAR, &coordtype));
4535     PetscCallMPI(MPI_Type_commit(&coordtype));
4536 #if defined(PETSC_USE_COMPLEX)
4537     {
4538       PetscScalar *svertexCoords;
4539       PetscInt     i;
4540       PetscCall(PetscMalloc1(numVertices * spaceDim, &svertexCoords));
4541       for (i = 0; i < numVertices * spaceDim; i++) svertexCoords[i] = vertexCoords[i];
4542       PetscCall(PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4543       PetscCall(PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4544       PetscCall(PetscFree(svertexCoords));
4545     }
4546 #else
4547     PetscCall(PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4548     PetscCall(PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4549 #endif
4550     PetscCallMPI(MPI_Type_free(&coordtype));
4551   }
4552   PetscCall(VecRestoreArray(coordinates, &coords));
4553   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4554   PetscCall(VecDestroy(&coordinates));
4555   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4556   PetscFunctionReturn(0);
4557 }
4558 
4559 /*@
4560   DMPlexCreateFromCellListParallelPetsc - Create distributed DMPLEX from a list of vertices for each cell (common mesh generator output)
4561 
4562   Input Parameters:
4563 + comm - The communicator
4564 . dim - The topological dimension of the mesh
4565 . numCells - The number of cells owned by this process
4566 . numVertices - The number of vertices owned by this process, or PETSC_DECIDE
4567 . NVertices - The global number of vertices, or PETSC_DECIDE
4568 . numCorners - The number of vertices for each cell
4569 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4570 . cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4571 . spaceDim - The spatial dimension used for coordinates
4572 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4573 
4574   Output Parameters:
4575 + dm - The DM
4576 . vertexSF - (Optional) SF describing complete vertex ownership
4577 - verticesAdjSaved - (Optional) vertex adjacency array
4578 
4579   Notes:
4580   This function is just a convenient sequence of DMCreate(), DMSetType(), DMSetDimension(),
4581   DMPlexBuildFromCellListParallel(), DMPlexInterpolate(), DMPlexBuildCoordinatesFromCellListParallel()
4582 
4583   See DMPlexBuildFromCellListParallel() for an example and details about the topology-related parameters.
4584   See DMPlexBuildCoordinatesFromCellListParallel() for details about the geometry-related parameters.
4585 
4586   Level: intermediate
4587 
4588 .seealso: `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
4589 @*/
4590 PetscErrorCode DMPlexCreateFromCellListParallelPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, PetscInt **verticesAdj, DM *dm)
4591 {
4592   PetscSF sfVert;
4593 
4594   PetscFunctionBegin;
4595   PetscCall(DMCreate(comm, dm));
4596   PetscCall(DMSetType(*dm, DMPLEX));
4597   PetscValidLogicalCollectiveInt(*dm, dim, 2);
4598   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
4599   PetscCall(DMSetDimension(*dm, dim));
4600   PetscCall(DMPlexBuildFromCellListParallel(*dm, numCells, numVertices, NVertices, numCorners, cells, &sfVert, verticesAdj));
4601   if (interpolate) {
4602     DM idm;
4603 
4604     PetscCall(DMPlexInterpolate(*dm, &idm));
4605     PetscCall(DMDestroy(dm));
4606     *dm = idm;
4607   }
4608   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
4609   if (vertexSF) *vertexSF = sfVert;
4610   else PetscCall(PetscSFDestroy(&sfVert));
4611   PetscFunctionReturn(0);
4612 }
4613 
4614 /*@C
4615   DMPlexBuildFromCellList - Build DMPLEX topology from a list of vertices for each cell (common mesh generator output)
4616 
4617   Input Parameters:
4618 + dm - The DM
4619 . numCells - The number of cells owned by this process
4620 . numVertices - The number of vertices owned by this process, or PETSC_DETERMINE
4621 . numCorners - The number of vertices for each cell
4622 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4623 
4624   Level: advanced
4625 
4626   Notes:
4627   Two triangles sharing a face
4628 $
4629 $        2
4630 $      / | \
4631 $     /  |  \
4632 $    /   |   \
4633 $   0  0 | 1  3
4634 $    \   |   /
4635 $     \  |  /
4636 $      \ | /
4637 $        1
4638 would have input
4639 $  numCells = 2, numVertices = 4
4640 $  cells = [0 1 2  1 3 2]
4641 $
4642 which would result in the DMPlex
4643 $
4644 $        4
4645 $      / | \
4646 $     /  |  \
4647 $    /   |   \
4648 $   2  0 | 1  5
4649 $    \   |   /
4650 $     \  |  /
4651 $      \ | /
4652 $        3
4653 
4654   If numVertices is PETSC_DETERMINE, it is computed by PETSc as the maximum vertex index in cells + 1.
4655 
4656   Not currently supported in Fortran.
4657 
4658 .seealso: `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListPetsc()`
4659 @*/
4660 PetscErrorCode DMPlexBuildFromCellList(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const PetscInt cells[])
4661 {
4662   PetscInt *cones, c, p, dim;
4663 
4664   PetscFunctionBegin;
4665   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4666   PetscCall(DMGetDimension(dm, &dim));
4667   /* Get/check global number of vertices */
4668   {
4669     PetscInt       NVerticesInCells, i;
4670     const PetscInt len = numCells * numCorners;
4671 
4672     /* NVerticesInCells = max(cells) + 1 */
4673     NVerticesInCells = PETSC_MIN_INT;
4674     for (i = 0; i < len; i++)
4675       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4676     ++NVerticesInCells;
4677 
4678     if (numVertices == PETSC_DECIDE) numVertices = NVerticesInCells;
4679     else
4680       PetscCheck(numVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, numVertices, NVerticesInCells);
4681   }
4682   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
4683   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
4684   PetscCall(DMSetUp(dm));
4685   PetscCall(DMPlexGetCones(dm, &cones));
4686   for (c = 0; c < numCells; ++c) {
4687     for (p = 0; p < numCorners; ++p) cones[c * numCorners + p] = cells[c * numCorners + p] + numCells;
4688   }
4689   PetscCall(DMPlexSymmetrize(dm));
4690   PetscCall(DMPlexStratify(dm));
4691   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4692   PetscFunctionReturn(0);
4693 }
4694 
4695 /*@C
4696   DMPlexBuildCoordinatesFromCellList - Build DM coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4697 
4698   Input Parameters:
4699 + dm - The DM
4700 . spaceDim - The spatial dimension used for coordinates
4701 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4702 
4703   Level: advanced
4704 
4705   Notes:
4706   Not currently supported in Fortran.
4707 
4708 .seealso: `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellList()`
4709 @*/
4710 PetscErrorCode DMPlexBuildCoordinatesFromCellList(DM dm, PetscInt spaceDim, const PetscReal vertexCoords[])
4711 {
4712   PetscSection coordSection;
4713   Vec          coordinates;
4714   DM           cdm;
4715   PetscScalar *coords;
4716   PetscInt     v, vStart, vEnd, d;
4717 
4718   PetscFunctionBegin;
4719   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4720   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
4721   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4722   PetscCall(DMSetCoordinateDim(dm, spaceDim));
4723   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4724   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4725   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
4726   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
4727   for (v = vStart; v < vEnd; ++v) {
4728     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
4729     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
4730   }
4731   PetscCall(PetscSectionSetUp(coordSection));
4732 
4733   PetscCall(DMGetCoordinateDM(dm, &cdm));
4734   PetscCall(DMCreateLocalVector(cdm, &coordinates));
4735   PetscCall(VecSetBlockSize(coordinates, spaceDim));
4736   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4737   PetscCall(VecGetArrayWrite(coordinates, &coords));
4738   for (v = 0; v < vEnd - vStart; ++v) {
4739     for (d = 0; d < spaceDim; ++d) coords[v * spaceDim + d] = vertexCoords[v * spaceDim + d];
4740   }
4741   PetscCall(VecRestoreArrayWrite(coordinates, &coords));
4742   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4743   PetscCall(VecDestroy(&coordinates));
4744   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4745   PetscFunctionReturn(0);
4746 }
4747 
4748 /*@
4749   DMPlexCreateFromCellListPetsc - Create DMPLEX from a list of vertices for each cell (common mesh generator output), but only process 0 takes in the input
4750 
4751   Collective on comm
4752 
4753   Input Parameters:
4754 + comm - The communicator
4755 . dim - The topological dimension of the mesh
4756 . numCells - The number of cells, only on process 0
4757 . numVertices - The number of vertices owned by this process, or PETSC_DECIDE, only on process 0
4758 . numCorners - The number of vertices for each cell, only on process 0
4759 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4760 . cells - An array of numCells*numCorners numbers, the vertices for each cell, only on process 0
4761 . spaceDim - The spatial dimension used for coordinates
4762 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex, only on process 0
4763 
4764   Output Parameter:
4765 . dm - The DM, which only has points on process 0
4766 
4767   Notes:
4768   This function is just a convenient sequence of DMCreate(), DMSetType(), DMSetDimension(), DMPlexBuildFromCellList(),
4769   DMPlexInterpolate(), DMPlexBuildCoordinatesFromCellList()
4770 
4771   See DMPlexBuildFromCellList() for an example and details about the topology-related parameters.
4772   See DMPlexBuildCoordinatesFromCellList() for details about the geometry-related parameters.
4773   See DMPlexCreateFromCellListParallelPetsc() for parallel input
4774 
4775   Level: intermediate
4776 
4777 .seealso: `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellList()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
4778 @*/
4779 PetscErrorCode DMPlexCreateFromCellListPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], DM *dm)
4780 {
4781   PetscMPIInt rank;
4782 
4783   PetscFunctionBegin;
4784   PetscCheck(dim, comm, PETSC_ERR_ARG_OUTOFRANGE, "This is not appropriate for 0-dimensional meshes. Consider either creating the DM using DMPlexCreateFromDAG(), by hand, or using DMSwarm.");
4785   PetscCallMPI(MPI_Comm_rank(comm, &rank));
4786   PetscCall(DMCreate(comm, dm));
4787   PetscCall(DMSetType(*dm, DMPLEX));
4788   PetscCall(DMSetDimension(*dm, dim));
4789   if (rank == 0) PetscCall(DMPlexBuildFromCellList(*dm, numCells, numVertices, numCorners, cells));
4790   else PetscCall(DMPlexBuildFromCellList(*dm, 0, 0, 0, NULL));
4791   if (interpolate) {
4792     DM idm;
4793 
4794     PetscCall(DMPlexInterpolate(*dm, &idm));
4795     PetscCall(DMDestroy(dm));
4796     *dm = idm;
4797   }
4798   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, vertexCoords));
4799   else PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, NULL));
4800   PetscFunctionReturn(0);
4801 }
4802 
4803 /*@
4804   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a DM
4805 
4806   Input Parameters:
4807 + dm - The empty DM object, usually from DMCreate() and DMSetDimension()
4808 . depth - The depth of the DAG
4809 . numPoints - Array of size depth + 1 containing the number of points at each depth
4810 . coneSize - The cone size of each point
4811 . cones - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
4812 . coneOrientations - The orientation of each cone point
4813 - vertexCoords - An array of numPoints[0]*spacedim numbers representing the coordinates of each vertex, with spacedim the value set via DMSetCoordinateDim()
4814 
4815   Output Parameter:
4816 . dm - The DM
4817 
4818   Note: Two triangles sharing a face would have input
4819 $  depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
4820 $  cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
4821 $ vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
4822 $
4823 which would result in the DMPlex
4824 $
4825 $        4
4826 $      / | \
4827 $     /  |  \
4828 $    /   |   \
4829 $   2  0 | 1  5
4830 $    \   |   /
4831 $     \  |  /
4832 $      \ | /
4833 $        3
4834 $
4835 $ Notice that all points are numbered consecutively, unlike DMPlexCreateFromCellListPetsc()
4836 
4837   Level: advanced
4838 
4839 .seealso: `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
4840 @*/
4841 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
4842 {
4843   Vec          coordinates;
4844   PetscSection coordSection;
4845   PetscScalar *coords;
4846   PetscInt     coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
4847 
4848   PetscFunctionBegin;
4849   PetscCall(DMGetDimension(dm, &dim));
4850   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
4851   PetscCheck(dimEmbed >= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Embedding dimension %" PetscInt_FMT " cannot be less than intrinsic dimension %" PetscInt_FMT, dimEmbed, dim);
4852   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
4853   PetscCall(DMPlexSetChart(dm, pStart, pEnd));
4854   for (p = pStart; p < pEnd; ++p) {
4855     PetscCall(DMPlexSetConeSize(dm, p, coneSize[p - pStart]));
4856     if (firstVertex < 0 && !coneSize[p - pStart]) firstVertex = p - pStart;
4857   }
4858   PetscCheck(firstVertex >= 0 || !numPoints[0], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Expected %" PetscInt_FMT " vertices but could not find any", numPoints[0]);
4859   PetscCall(DMSetUp(dm)); /* Allocate space for cones */
4860   for (p = pStart, off = 0; p < pEnd; off += coneSize[p - pStart], ++p) {
4861     PetscCall(DMPlexSetCone(dm, p, &cones[off]));
4862     PetscCall(DMPlexSetConeOrientation(dm, p, &coneOrientations[off]));
4863   }
4864   PetscCall(DMPlexSymmetrize(dm));
4865   PetscCall(DMPlexStratify(dm));
4866   /* Build coordinates */
4867   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4868   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4869   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dimEmbed));
4870   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numPoints[0]));
4871   for (v = firstVertex; v < firstVertex + numPoints[0]; ++v) {
4872     PetscCall(PetscSectionSetDof(coordSection, v, dimEmbed));
4873     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed));
4874   }
4875   PetscCall(PetscSectionSetUp(coordSection));
4876   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4877   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
4878   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4879   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4880   PetscCall(VecSetBlockSize(coordinates, dimEmbed));
4881   PetscCall(VecSetType(coordinates, VECSTANDARD));
4882   if (vertexCoords) {
4883     PetscCall(VecGetArray(coordinates, &coords));
4884     for (v = 0; v < numPoints[0]; ++v) {
4885       PetscInt off;
4886 
4887       PetscCall(PetscSectionGetOffset(coordSection, v + firstVertex, &off));
4888       for (d = 0; d < dimEmbed; ++d) coords[off + d] = vertexCoords[v * dimEmbed + d];
4889     }
4890   }
4891   PetscCall(VecRestoreArray(coordinates, &coords));
4892   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4893   PetscCall(VecDestroy(&coordinates));
4894   PetscFunctionReturn(0);
4895 }
4896 
4897 /*@C
4898   DMPlexCreateCellVertexFromFile - Create a DMPlex mesh from a simple cell-vertex file.
4899 
4900 + comm        - The MPI communicator
4901 . filename    - Name of the .dat file
4902 - interpolate - Create faces and edges in the mesh
4903 
4904   Output Parameter:
4905 . dm  - The DM object representing the mesh
4906 
4907   Note: The format is the simplest possible:
4908 $ Ne
4909 $ v0 v1 ... vk
4910 $ Nv
4911 $ x y z marker
4912 
4913   Level: beginner
4914 
4915 .seealso: `DMPlexCreateFromFile()`, `DMPlexCreateMedFromFile()`, `DMPlexCreateGmsh()`, `DMPlexCreate()`
4916 @*/
4917 PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
4918 {
4919   DMLabel      marker;
4920   PetscViewer  viewer;
4921   Vec          coordinates;
4922   PetscSection coordSection;
4923   PetscScalar *coords;
4924   char         line[PETSC_MAX_PATH_LEN];
4925   PetscInt     dim = 3, cdim = 3, coordSize, v, c, d;
4926   PetscMPIInt  rank;
4927   int          snum, Nv, Nc, Ncn, Nl;
4928 
4929   PetscFunctionBegin;
4930   PetscCallMPI(MPI_Comm_rank(comm, &rank));
4931   PetscCall(PetscViewerCreate(comm, &viewer));
4932   PetscCall(PetscViewerSetType(viewer, PETSCVIEWERASCII));
4933   PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
4934   PetscCall(PetscViewerFileSetName(viewer, filename));
4935   if (rank == 0) {
4936     PetscCall(PetscViewerRead(viewer, line, 4, NULL, PETSC_STRING));
4937     snum = sscanf(line, "%d %d %d %d", &Nc, &Nv, &Ncn, &Nl);
4938     PetscCheck(snum == 4, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4939   } else {
4940     Nc = Nv = Ncn = Nl = 0;
4941   }
4942   PetscCall(DMCreate(comm, dm));
4943   PetscCall(DMSetType(*dm, DMPLEX));
4944   PetscCall(DMPlexSetChart(*dm, 0, Nc + Nv));
4945   PetscCall(DMSetDimension(*dm, dim));
4946   PetscCall(DMSetCoordinateDim(*dm, cdim));
4947   /* Read topology */
4948   if (rank == 0) {
4949     char     format[PETSC_MAX_PATH_LEN];
4950     PetscInt cone[8];
4951     int      vbuf[8], v;
4952 
4953     for (c = 0; c < Ncn; ++c) {
4954       format[c * 3 + 0] = '%';
4955       format[c * 3 + 1] = 'd';
4956       format[c * 3 + 2] = ' ';
4957     }
4958     format[Ncn * 3 - 1] = '\0';
4959     for (c = 0; c < Nc; ++c) PetscCall(DMPlexSetConeSize(*dm, c, Ncn));
4960     PetscCall(DMSetUp(*dm));
4961     for (c = 0; c < Nc; ++c) {
4962       PetscCall(PetscViewerRead(viewer, line, Ncn, NULL, PETSC_STRING));
4963       switch (Ncn) {
4964       case 2:
4965         snum = sscanf(line, format, &vbuf[0], &vbuf[1]);
4966         break;
4967       case 3:
4968         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2]);
4969         break;
4970       case 4:
4971         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3]);
4972         break;
4973       case 6:
4974         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5]);
4975         break;
4976       case 8:
4977         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);
4978         break;
4979       default:
4980         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No cell shape with %d vertices", Ncn);
4981       }
4982       PetscCheck(snum == Ncn, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4983       for (v = 0; v < Ncn; ++v) cone[v] = vbuf[v] + Nc;
4984       /* Hexahedra are inverted */
4985       if (Ncn == 8) {
4986         PetscInt tmp = cone[1];
4987         cone[1]      = cone[3];
4988         cone[3]      = tmp;
4989       }
4990       PetscCall(DMPlexSetCone(*dm, c, cone));
4991     }
4992   }
4993   PetscCall(DMPlexSymmetrize(*dm));
4994   PetscCall(DMPlexStratify(*dm));
4995   /* Read coordinates */
4996   PetscCall(DMGetCoordinateSection(*dm, &coordSection));
4997   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4998   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
4999   PetscCall(PetscSectionSetChart(coordSection, Nc, Nc + Nv));
5000   for (v = Nc; v < Nc + Nv; ++v) {
5001     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
5002     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
5003   }
5004   PetscCall(PetscSectionSetUp(coordSection));
5005   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
5006   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
5007   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5008   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
5009   PetscCall(VecSetBlockSize(coordinates, cdim));
5010   PetscCall(VecSetType(coordinates, VECSTANDARD));
5011   PetscCall(VecGetArray(coordinates, &coords));
5012   if (rank == 0) {
5013     char   format[PETSC_MAX_PATH_LEN];
5014     double x[3];
5015     int    l, val[3];
5016 
5017     if (Nl) {
5018       for (l = 0; l < Nl; ++l) {
5019         format[l * 3 + 0] = '%';
5020         format[l * 3 + 1] = 'd';
5021         format[l * 3 + 2] = ' ';
5022       }
5023       format[Nl * 3 - 1] = '\0';
5024       PetscCall(DMCreateLabel(*dm, "marker"));
5025       PetscCall(DMGetLabel(*dm, "marker", &marker));
5026     }
5027     for (v = 0; v < Nv; ++v) {
5028       PetscCall(PetscViewerRead(viewer, line, 3 + Nl, NULL, PETSC_STRING));
5029       snum = sscanf(line, "%lg %lg %lg", &x[0], &x[1], &x[2]);
5030       PetscCheck(snum == 3, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5031       switch (Nl) {
5032       case 0:
5033         snum = 0;
5034         break;
5035       case 1:
5036         snum = sscanf(line, format, &val[0]);
5037         break;
5038       case 2:
5039         snum = sscanf(line, format, &val[0], &val[1]);
5040         break;
5041       case 3:
5042         snum = sscanf(line, format, &val[0], &val[1], &val[2]);
5043         break;
5044       default:
5045         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Request support for %d labels", Nl);
5046       }
5047       PetscCheck(snum == Nl, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5048       for (d = 0; d < cdim; ++d) coords[v * cdim + d] = x[d];
5049       for (l = 0; l < Nl; ++l) PetscCall(DMLabelSetValue(marker, v + Nc, val[l]));
5050     }
5051   }
5052   PetscCall(VecRestoreArray(coordinates, &coords));
5053   PetscCall(DMSetCoordinatesLocal(*dm, coordinates));
5054   PetscCall(VecDestroy(&coordinates));
5055   PetscCall(PetscViewerDestroy(&viewer));
5056   if (interpolate) {
5057     DM      idm;
5058     DMLabel bdlabel;
5059 
5060     PetscCall(DMPlexInterpolate(*dm, &idm));
5061     PetscCall(DMDestroy(dm));
5062     *dm = idm;
5063 
5064     if (!Nl) {
5065       PetscCall(DMCreateLabel(*dm, "marker"));
5066       PetscCall(DMGetLabel(*dm, "marker", &bdlabel));
5067       PetscCall(DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel));
5068       PetscCall(DMPlexLabelComplete(*dm, bdlabel));
5069     }
5070   }
5071   PetscFunctionReturn(0);
5072 }
5073 
5074 /*@C
5075   DMPlexCreateFromFile - This takes a filename and produces a DM
5076 
5077   Input Parameters:
5078 + comm - The communicator
5079 . filename - A file name
5080 . plexname - The object name of the resulting DM, also used for intra-datafile lookup by some formats
5081 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
5082 
5083   Output Parameter:
5084 . dm - The DM
5085 
5086   Options Database Keys:
5087 . -dm_plex_create_from_hdf5_xdmf - use the PETSC_VIEWER_HDF5_XDMF format for reading HDF5
5088 
5089   Use -dm_plex_create_ prefix to pass options to the internal PetscViewer, e.g.
5090 $ -dm_plex_create_viewer_hdf5_collective
5091 
5092   Notes:
5093   Using PETSCVIEWERHDF5 type with PETSC_VIEWER_HDF5_PETSC format, one can save multiple DMPlex
5094   meshes in a single HDF5 file. This in turn requires one to name the DMPlex object with PetscObjectSetName()
5095   before saving it with DMView() and before loading it with DMLoad() for identification of the mesh object.
5096   The input parameter name is thus used to name the DMPlex object when DMPlexCreateFromFile() internally
5097   calls DMLoad(). Currently, name is ignored for other viewer types and/or formats.
5098 
5099   Level: beginner
5100 
5101 .seealso: `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`, `PetscObjectSetName()`, `DMView()`, `DMLoad()`
5102 @*/
5103 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], const char plexname[], PetscBool interpolate, DM *dm)
5104 {
5105   const char  extGmsh[]      = ".msh";
5106   const char  extGmsh2[]     = ".msh2";
5107   const char  extGmsh4[]     = ".msh4";
5108   const char  extCGNS[]      = ".cgns";
5109   const char  extExodus[]    = ".exo";
5110   const char  extExodus_e[]  = ".e";
5111   const char  extGenesis[]   = ".gen";
5112   const char  extFluent[]    = ".cas";
5113   const char  extHDF5[]      = ".h5";
5114   const char  extMed[]       = ".med";
5115   const char  extPLY[]       = ".ply";
5116   const char  extEGADSLite[] = ".egadslite";
5117   const char  extEGADS[]     = ".egads";
5118   const char  extIGES[]      = ".igs";
5119   const char  extSTEP[]      = ".stp";
5120   const char  extCV[]        = ".dat";
5121   size_t      len;
5122   PetscBool   isGmsh, isGmsh2, isGmsh4, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isMed, isPLY, isEGADSLite, isEGADS, isIGES, isSTEP, isCV;
5123   PetscMPIInt rank;
5124 
5125   PetscFunctionBegin;
5126   PetscValidCharPointer(filename, 2);
5127   if (plexname) PetscValidCharPointer(plexname, 3);
5128   PetscValidPointer(dm, 5);
5129   PetscCall(DMInitializePackage());
5130   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5131   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5132   PetscCall(PetscStrlen(filename, &len));
5133   PetscCheck(len, comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
5134 
5135 #define CheckExtension(extension__, is_extension__) \
5136   do { \
5137     PetscAssert(sizeof(extension__), comm, PETSC_ERR_PLIB, "Zero-size extension: %s", extension__); \
5138     /* don't count the null-terminator at the end */ \
5139     const size_t ext_len = sizeof(extension__) - 1; \
5140     if (len < ext_len) { \
5141       is_extension__ = PETSC_FALSE; \
5142     } else { \
5143       PetscCall(PetscStrncmp(filename + len - ext_len, extension__, ext_len, &is_extension__)); \
5144     } \
5145   } while (0)
5146 
5147   CheckExtension(extGmsh, isGmsh);
5148   CheckExtension(extGmsh2, isGmsh2);
5149   CheckExtension(extGmsh4, isGmsh4);
5150   CheckExtension(extCGNS, isCGNS);
5151   CheckExtension(extExodus, isExodus);
5152   if (!isExodus) CheckExtension(extExodus_e, isExodus);
5153   CheckExtension(extGenesis, isGenesis);
5154   CheckExtension(extFluent, isFluent);
5155   CheckExtension(extHDF5, isHDF5);
5156   CheckExtension(extMed, isMed);
5157   CheckExtension(extPLY, isPLY);
5158   CheckExtension(extEGADSLite, isEGADSLite);
5159   CheckExtension(extEGADS, isEGADS);
5160   CheckExtension(extIGES, isIGES);
5161   CheckExtension(extSTEP, isSTEP);
5162   CheckExtension(extCV, isCV);
5163 
5164 #undef CheckExtension
5165 
5166   if (isGmsh || isGmsh2 || isGmsh4) {
5167     PetscCall(DMPlexCreateGmshFromFile(comm, filename, interpolate, dm));
5168   } else if (isCGNS) {
5169     PetscCall(DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm));
5170   } else if (isExodus || isGenesis) {
5171     PetscCall(DMPlexCreateExodusFromFile(comm, filename, interpolate, dm));
5172   } else if (isFluent) {
5173     PetscCall(DMPlexCreateFluentFromFile(comm, filename, interpolate, dm));
5174   } else if (isHDF5) {
5175     PetscBool   load_hdf5_xdmf = PETSC_FALSE;
5176     PetscViewer viewer;
5177 
5178     /* PETSC_VIEWER_HDF5_XDMF is used if the filename ends with .xdmf.h5, or if -dm_plex_create_from_hdf5_xdmf option is present */
5179     PetscCall(PetscStrncmp(&filename[PetscMax(0, len - 8)], ".xdmf", 5, &load_hdf5_xdmf));
5180     PetscCall(PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &load_hdf5_xdmf, NULL));
5181     PetscCall(PetscViewerCreate(comm, &viewer));
5182     PetscCall(PetscViewerSetType(viewer, PETSCVIEWERHDF5));
5183     PetscCall(PetscViewerSetOptionsPrefix(viewer, "dm_plex_create_"));
5184     PetscCall(PetscViewerSetFromOptions(viewer));
5185     PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
5186     PetscCall(PetscViewerFileSetName(viewer, filename));
5187 
5188     PetscCall(DMCreate(comm, dm));
5189     PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5190     PetscCall(DMSetType(*dm, DMPLEX));
5191     if (load_hdf5_xdmf) PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF));
5192     PetscCall(DMLoad(*dm, viewer));
5193     if (load_hdf5_xdmf) PetscCall(PetscViewerPopFormat(viewer));
5194     PetscCall(PetscViewerDestroy(&viewer));
5195 
5196     if (interpolate) {
5197       DM idm;
5198 
5199       PetscCall(DMPlexInterpolate(*dm, &idm));
5200       PetscCall(DMDestroy(dm));
5201       *dm = idm;
5202     }
5203   } else if (isMed) {
5204     PetscCall(DMPlexCreateMedFromFile(comm, filename, interpolate, dm));
5205   } else if (isPLY) {
5206     PetscCall(DMPlexCreatePLYFromFile(comm, filename, interpolate, dm));
5207   } else if (isEGADSLite || isEGADS || isIGES || isSTEP) {
5208     if (isEGADSLite) PetscCall(DMPlexCreateEGADSLiteFromFile(comm, filename, dm));
5209     else PetscCall(DMPlexCreateEGADSFromFile(comm, filename, dm));
5210     if (!interpolate) {
5211       DM udm;
5212 
5213       PetscCall(DMPlexUninterpolate(*dm, &udm));
5214       PetscCall(DMDestroy(dm));
5215       *dm = udm;
5216     }
5217   } else if (isCV) {
5218     PetscCall(DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm));
5219   } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
5220   PetscCall(PetscStrlen(plexname, &len));
5221   if (len) PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5222   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5223   PetscFunctionReturn(0);
5224 }
5225 /*@C
5226   DMPlexCreateEphemeral - This takes a `DMPlexTransform` and a base `DMPlex` and produces an ephemeral `DM`, meaning one that is created on the fly in response to queries.
5227 
5228   Input Parameter:
5229 . tr - The `DMPlexTransform`
5230 
5231   Output Parameter:
5232 . dm - The `DM`
5233 
5234   Notes:
5235   An emphemeral mesh is one that is not stored concretely, as in the default Plex implementation, but rather is produced on the fly in response to queries, using information from the transform and the base mesh.
5236 
5237   Level: beginner
5238 
5239 .seealso: `DMPlexCreateFromFile`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
5240 @*/
5241 PetscErrorCode DMPlexCreateEphemeral(DMPlexTransform tr, DM *dm)
5242 {
5243   DM       bdm;
5244   PetscInt Nl;
5245 
5246   PetscFunctionBegin;
5247   PetscCall(DMCreate(PetscObjectComm((PetscObject)tr), dm));
5248   PetscCall(DMSetType(*dm, DMPLEX));
5249   PetscCall(DMSetFromOptions(*dm));
5250 
5251   PetscCall(PetscObjectReference((PetscObject)tr));
5252   PetscCall(DMPlexTransformDestroy(&((DM_Plex *)(*dm)->data)->tr));
5253   ((DM_Plex *)(*dm)->data)->tr = tr;
5254 
5255   PetscCall(DMPlexTransformGetDM(tr, &bdm));
5256   PetscCall(DMGetNumLabels(bdm, &Nl));
5257   for (PetscInt l = 0; l < Nl; ++l) {
5258     DMLabel     label, labelNew;
5259     const char *lname;
5260     PetscBool   isDepth, isCellType;
5261 
5262     PetscCall(DMGetLabelName(bdm, l, &lname));
5263     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
5264     if (isDepth) continue;
5265     PetscCall(PetscStrcmp(lname, "celltype", &isCellType));
5266     if (isCellType) continue;
5267     PetscCall(DMCreateLabel(*dm, lname));
5268     PetscCall(DMGetLabel(bdm, lname, &label));
5269     PetscCall(DMGetLabel(*dm, lname, &labelNew));
5270     PetscCall(DMLabelSetType(labelNew, DMLABELEPHEMERAL));
5271     PetscCall(DMLabelEphemeralSetLabel(labelNew, label));
5272     PetscCall(DMLabelEphemeralSetTransform(labelNew, tr));
5273     PetscCall(DMLabelSetUp(labelNew));
5274   }
5275   PetscFunctionReturn(0);
5276 }
5277