xref: /petsc/src/dm/impls/plex/plexcreate.c (revision 9371c9d470a9602b6d10a8bf50c9b2280a79e45a)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h> /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>   /*I   "petscdmplex.h"   I*/
4 #include <petscsf.h>
5 #include <petscdmplextransform.h>
6 #include <petsc/private/kernels/blockmatmult.h>
7 #include <petsc/private/kernels/blockinvert.h>
8 
9 PetscLogEvent DMPLEX_CreateFromFile, DMPLEX_BuildFromCellList, DMPLEX_BuildCoordinatesFromCellList;
10 
11 /* External function declarations here */
12 static PetscErrorCode DMInitialize_Plex(DM dm);
13 
14 /* This copies internal things in the Plex structure that we generally want when making a new, related Plex */
15 PetscErrorCode DMPlexCopy_Internal(DM dmin, PetscBool copyPeriodicity, PetscBool copyOverlap, DM dmout) {
16   const PetscReal         *maxCell, *Lstart, *L;
17   PetscBool                dist;
18   DMPlexReorderDefaultFlag reorder;
19 
20   PetscFunctionBegin;
21   if (copyPeriodicity) {
22     PetscCall(DMGetPeriodicity(dmin, &maxCell, &Lstart, &L));
23     PetscCall(DMSetPeriodicity(dmout, maxCell, Lstart, L));
24   }
25   PetscCall(DMPlexDistributeGetDefault(dmin, &dist));
26   PetscCall(DMPlexDistributeSetDefault(dmout, dist));
27   PetscCall(DMPlexReorderGetDefault(dmin, &reorder));
28   PetscCall(DMPlexReorderSetDefault(dmout, reorder));
29   ((DM_Plex *)dmout->data)->useHashLocation = ((DM_Plex *)dmin->data)->useHashLocation;
30   if (copyOverlap) PetscCall(DMPlexSetOverlap_Plex(dmout, dmin, 0));
31   PetscFunctionReturn(0);
32 }
33 
34 /* Replace dm with the contents of ndm, and then destroy ndm
35    - Share the DM_Plex structure
36    - Share the coordinates
37    - Share the SF
38 */
39 static PetscErrorCode DMPlexReplace_Static(DM dm, DM *ndm) {
40   PetscSF          sf;
41   DM               dmNew = *ndm, coordDM, coarseDM;
42   Vec              coords;
43   const PetscReal *maxCell, *Lstart, *L;
44   PetscInt         dim, cdim;
45 
46   PetscFunctionBegin;
47   if (dm == dmNew) {
48     PetscCall(DMDestroy(ndm));
49     PetscFunctionReturn(0);
50   }
51   dm->setupcalled = dmNew->setupcalled;
52   PetscCall(DMGetDimension(dmNew, &dim));
53   PetscCall(DMSetDimension(dm, dim));
54   PetscCall(DMGetCoordinateDim(dmNew, &cdim));
55   PetscCall(DMSetCoordinateDim(dm, cdim));
56   PetscCall(DMGetPointSF(dmNew, &sf));
57   PetscCall(DMSetPointSF(dm, sf));
58   PetscCall(DMGetCoordinateDM(dmNew, &coordDM));
59   PetscCall(DMGetCoordinatesLocal(dmNew, &coords));
60   PetscCall(DMSetCoordinateDM(dm, coordDM));
61   PetscCall(DMSetCoordinatesLocal(dm, coords));
62   PetscCall(DMGetCellCoordinateDM(dmNew, &coordDM));
63   PetscCall(DMGetCellCoordinatesLocal(dmNew, &coords));
64   PetscCall(DMSetCellCoordinateDM(dm, coordDM));
65   PetscCall(DMSetCellCoordinatesLocal(dm, coords));
66   /* Do not want to create the coordinate field if it does not already exist, so do not call DMGetCoordinateField() */
67   PetscCall(DMFieldDestroy(&dm->coordinates[0].field));
68   dm->coordinates[0].field            = dmNew->coordinates[0].field;
69   ((DM_Plex *)dmNew->data)->coordFunc = ((DM_Plex *)dm->data)->coordFunc;
70   PetscCall(DMGetPeriodicity(dmNew, &maxCell, &Lstart, &L));
71   PetscCall(DMSetPeriodicity(dm, maxCell, Lstart, L));
72   PetscCall(DMDestroy_Plex(dm));
73   PetscCall(DMInitialize_Plex(dm));
74   dm->data = dmNew->data;
75   ((DM_Plex *)dmNew->data)->refct++;
76   PetscCall(DMDestroyLabelLinkList_Internal(dm));
77   PetscCall(DMCopyLabels(dmNew, dm, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL));
78   PetscCall(DMGetCoarseDM(dmNew, &coarseDM));
79   PetscCall(DMSetCoarseDM(dm, coarseDM));
80   PetscCall(DMDestroy(ndm));
81   PetscFunctionReturn(0);
82 }
83 
84 /* Swap dm with the contents of dmNew
85    - Swap the DM_Plex structure
86    - Swap the coordinates
87    - Swap the point PetscSF
88 */
89 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB) {
90   DM          coordDMA, coordDMB;
91   Vec         coordsA, coordsB;
92   PetscSF     sfA, sfB;
93   DMField     fieldTmp;
94   void       *tmp;
95   DMLabelLink listTmp;
96   DMLabel     depthTmp;
97   PetscInt    tmpI;
98 
99   PetscFunctionBegin;
100   if (dmA == dmB) PetscFunctionReturn(0);
101   PetscCall(DMGetPointSF(dmA, &sfA));
102   PetscCall(DMGetPointSF(dmB, &sfB));
103   PetscCall(PetscObjectReference((PetscObject)sfA));
104   PetscCall(DMSetPointSF(dmA, sfB));
105   PetscCall(DMSetPointSF(dmB, sfA));
106   PetscCall(PetscObjectDereference((PetscObject)sfA));
107 
108   PetscCall(DMGetCoordinateDM(dmA, &coordDMA));
109   PetscCall(DMGetCoordinateDM(dmB, &coordDMB));
110   PetscCall(PetscObjectReference((PetscObject)coordDMA));
111   PetscCall(DMSetCoordinateDM(dmA, coordDMB));
112   PetscCall(DMSetCoordinateDM(dmB, coordDMA));
113   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
114 
115   PetscCall(DMGetCoordinatesLocal(dmA, &coordsA));
116   PetscCall(DMGetCoordinatesLocal(dmB, &coordsB));
117   PetscCall(PetscObjectReference((PetscObject)coordsA));
118   PetscCall(DMSetCoordinatesLocal(dmA, coordsB));
119   PetscCall(DMSetCoordinatesLocal(dmB, coordsA));
120   PetscCall(PetscObjectDereference((PetscObject)coordsA));
121 
122   PetscCall(DMGetCellCoordinateDM(dmA, &coordDMA));
123   PetscCall(DMGetCellCoordinateDM(dmB, &coordDMB));
124   PetscCall(PetscObjectReference((PetscObject)coordDMA));
125   PetscCall(DMSetCellCoordinateDM(dmA, coordDMB));
126   PetscCall(DMSetCellCoordinateDM(dmB, coordDMA));
127   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
128 
129   PetscCall(DMGetCellCoordinatesLocal(dmA, &coordsA));
130   PetscCall(DMGetCellCoordinatesLocal(dmB, &coordsB));
131   PetscCall(PetscObjectReference((PetscObject)coordsA));
132   PetscCall(DMSetCellCoordinatesLocal(dmA, coordsB));
133   PetscCall(DMSetCellCoordinatesLocal(dmB, coordsA));
134   PetscCall(PetscObjectDereference((PetscObject)coordsA));
135 
136   fieldTmp                  = dmA->coordinates[0].field;
137   dmA->coordinates[0].field = dmB->coordinates[0].field;
138   dmB->coordinates[0].field = fieldTmp;
139   fieldTmp                  = dmA->coordinates[1].field;
140   dmA->coordinates[1].field = dmB->coordinates[1].field;
141   dmB->coordinates[1].field = fieldTmp;
142   tmp                       = dmA->data;
143   dmA->data                 = dmB->data;
144   dmB->data                 = tmp;
145   listTmp                   = dmA->labels;
146   dmA->labels               = dmB->labels;
147   dmB->labels               = listTmp;
148   depthTmp                  = dmA->depthLabel;
149   dmA->depthLabel           = dmB->depthLabel;
150   dmB->depthLabel           = depthTmp;
151   depthTmp                  = dmA->celltypeLabel;
152   dmA->celltypeLabel        = dmB->celltypeLabel;
153   dmB->celltypeLabel        = depthTmp;
154   tmpI                      = dmA->levelup;
155   dmA->levelup              = dmB->levelup;
156   dmB->levelup              = tmpI;
157   PetscFunctionReturn(0);
158 }
159 
160 static PetscErrorCode DMPlexInterpolateInPlace_Internal(DM dm) {
161   DM idm;
162 
163   PetscFunctionBegin;
164   PetscCall(DMPlexInterpolate(dm, &idm));
165   PetscCall(DMPlexCopyCoordinates(dm, idm));
166   PetscCall(DMPlexReplace_Static(dm, &idm));
167   PetscFunctionReturn(0);
168 }
169 
170 /*@C
171   DMPlexCreateCoordinateSpace - Creates a finite element space for the coordinates
172 
173   Collective
174 
175   Input Parameters:
176 + DM        - The DM
177 . degree    - The degree of the finite element or PETSC_DECIDE
178 - coordFunc - An optional function to map new points from refinement to the surface
179 
180   Level: advanced
181 
182 .seealso: `PetscFECreateLagrange()`, `DMGetCoordinateDM()`
183 @*/
184 PetscErrorCode DMPlexCreateCoordinateSpace(DM dm, PetscInt degree, PetscPointFunc coordFunc) {
185   DM_Plex     *mesh = (DM_Plex *)dm->data;
186   DM           cdm;
187   PetscDS      cds;
188   PetscFE      fe;
189   PetscClassId id;
190 
191   PetscFunctionBegin;
192   PetscCall(DMGetCoordinateDM(dm, &cdm));
193   PetscCall(DMGetDS(cdm, &cds));
194   PetscCall(PetscDSGetDiscretization(cds, 0, (PetscObject *)&fe));
195   PetscCall(PetscObjectGetClassId((PetscObject)fe, &id));
196   if (id != PETSCFE_CLASSID) {
197     PetscBool simplex;
198     PetscInt  dim, dE, qorder;
199 
200     PetscCall(DMGetDimension(dm, &dim));
201     PetscCall(DMGetCoordinateDim(dm, &dE));
202     qorder = degree;
203     PetscObjectOptionsBegin((PetscObject)cdm);
204     PetscCall(PetscOptionsBoundedInt("-coord_dm_default_quadrature_order", "Quadrature order is one less than quadrature points per edge", "DMPlexCreateCoordinateSpace", qorder, &qorder, NULL, 0));
205     PetscOptionsEnd();
206     if (degree == PETSC_DECIDE) fe = NULL;
207     else {
208       PetscCall(DMPlexIsSimplex(dm, &simplex));
209       PetscCall(PetscFECreateLagrange(PETSC_COMM_SELF, dim, dE, simplex, degree, qorder, &fe));
210     }
211     PetscCall(DMProjectCoordinates(dm, fe));
212     PetscCall(PetscFEDestroy(&fe));
213   }
214   mesh->coordFunc = coordFunc;
215   PetscFunctionReturn(0);
216 }
217 
218 /*@
219   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
220 
221   Collective
222 
223   Input Parameters:
224 + comm - The communicator for the DM object
225 . dim - The spatial dimension
226 . simplex - Flag for simplicial cells, otherwise they are tensor product cells
227 . interpolate - Flag to create intermediate mesh pieces (edges, faces)
228 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
229 
230   Output Parameter:
231 . dm - The DM object
232 
233   Level: beginner
234 
235 .seealso: `DMSetType()`, `DMCreate()`
236 @*/
237 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscReal refinementLimit, DM *newdm) {
238   DM          dm;
239   PetscMPIInt rank;
240 
241   PetscFunctionBegin;
242   PetscCall(DMCreate(comm, &dm));
243   PetscCall(DMSetType(dm, DMPLEX));
244   PetscCall(DMSetDimension(dm, dim));
245   PetscCallMPI(MPI_Comm_rank(comm, &rank));
246   switch (dim) {
247   case 2:
248     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "triangular"));
249     else PetscCall(PetscObjectSetName((PetscObject)dm, "quadrilateral"));
250     break;
251   case 3:
252     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "tetrahedral"));
253     else PetscCall(PetscObjectSetName((PetscObject)dm, "hexahedral"));
254     break;
255   default: SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
256   }
257   if (rank) {
258     PetscInt numPoints[2] = {0, 0};
259     PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL));
260   } else {
261     switch (dim) {
262     case 2:
263       if (simplex) {
264         PetscInt    numPoints[2]        = {4, 2};
265         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
266         PetscInt    cones[6]            = {2, 3, 4, 5, 4, 3};
267         PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
268         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
269 
270         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
271       } else {
272         PetscInt    numPoints[2]        = {6, 2};
273         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
274         PetscInt    cones[8]            = {2, 3, 4, 5, 3, 6, 7, 4};
275         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
276         PetscScalar vertexCoords[12]    = {-1.0, -0.5, 0.0, -0.5, 0.0, 0.5, -1.0, 0.5, 1.0, -0.5, 1.0, 0.5};
277 
278         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
279       }
280       break;
281     case 3:
282       if (simplex) {
283         PetscInt    numPoints[2]        = {5, 2};
284         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
285         PetscInt    cones[8]            = {4, 3, 5, 2, 5, 3, 4, 6};
286         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
287         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0};
288 
289         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
290       } else {
291         PetscInt    numPoints[2]         = {12, 2};
292         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
293         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9, 5, 4, 10, 11, 7, 12, 13, 8};
294         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
295         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5, -1.0, 0.5, -0.5, 0.0, 0.5, -0.5, 0.0, -0.5, -0.5, -1.0, -0.5, 0.5, 0.0, -0.5, 0.5, 0.0, 0.5, 0.5, -1.0, 0.5, 0.5, 1.0, 0.5, -0.5, 1.0, -0.5, -0.5, 1.0, -0.5, 0.5, 1.0, 0.5, 0.5};
296 
297         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
298       }
299       break;
300     default: SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
301     }
302   }
303   *newdm = dm;
304   if (refinementLimit > 0.0) {
305     DM          rdm;
306     const char *name;
307 
308     PetscCall(DMPlexSetRefinementUniform(*newdm, PETSC_FALSE));
309     PetscCall(DMPlexSetRefinementLimit(*newdm, refinementLimit));
310     PetscCall(DMRefine(*newdm, comm, &rdm));
311     PetscCall(PetscObjectGetName((PetscObject)*newdm, &name));
312     PetscCall(PetscObjectSetName((PetscObject)rdm, name));
313     PetscCall(DMDestroy(newdm));
314     *newdm = rdm;
315   }
316   if (interpolate) {
317     DM idm;
318 
319     PetscCall(DMPlexInterpolate(*newdm, &idm));
320     PetscCall(DMDestroy(newdm));
321     *newdm = idm;
322   }
323   PetscFunctionReturn(0);
324 }
325 
326 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[]) {
327   const PetscInt numVertices    = 2;
328   PetscInt       markerRight    = 1;
329   PetscInt       markerLeft     = 1;
330   PetscBool      markerSeparate = PETSC_FALSE;
331   Vec            coordinates;
332   PetscSection   coordSection;
333   PetscScalar   *coords;
334   PetscInt       coordSize;
335   PetscMPIInt    rank;
336   PetscInt       cdim = 1, v;
337 
338   PetscFunctionBegin;
339   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
340   if (markerSeparate) {
341     markerRight = 2;
342     markerLeft  = 1;
343   }
344   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
345   if (rank == 0) {
346     PetscCall(DMPlexSetChart(dm, 0, numVertices));
347     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
348     PetscCall(DMSetLabelValue(dm, "marker", 0, markerLeft));
349     PetscCall(DMSetLabelValue(dm, "marker", 1, markerRight));
350   }
351   PetscCall(DMPlexSymmetrize(dm));
352   PetscCall(DMPlexStratify(dm));
353   /* Build coordinates */
354   PetscCall(DMSetCoordinateDim(dm, cdim));
355   PetscCall(DMGetCoordinateSection(dm, &coordSection));
356   PetscCall(PetscSectionSetNumFields(coordSection, 1));
357   PetscCall(PetscSectionSetChart(coordSection, 0, numVertices));
358   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
359   for (v = 0; v < numVertices; ++v) {
360     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
361     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
362   }
363   PetscCall(PetscSectionSetUp(coordSection));
364   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
365   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
366   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
367   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
368   PetscCall(VecSetBlockSize(coordinates, cdim));
369   PetscCall(VecSetType(coordinates, VECSTANDARD));
370   PetscCall(VecGetArray(coordinates, &coords));
371   coords[0] = lower[0];
372   coords[1] = upper[0];
373   PetscCall(VecRestoreArray(coordinates, &coords));
374   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
375   PetscCall(VecDestroy(&coordinates));
376   PetscFunctionReturn(0);
377 }
378 
379 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[]) {
380   const PetscInt numVertices    = (edges[0] + 1) * (edges[1] + 1);
381   const PetscInt numEdges       = edges[0] * (edges[1] + 1) + (edges[0] + 1) * edges[1];
382   PetscInt       markerTop      = 1;
383   PetscInt       markerBottom   = 1;
384   PetscInt       markerRight    = 1;
385   PetscInt       markerLeft     = 1;
386   PetscBool      markerSeparate = PETSC_FALSE;
387   Vec            coordinates;
388   PetscSection   coordSection;
389   PetscScalar   *coords;
390   PetscInt       coordSize;
391   PetscMPIInt    rank;
392   PetscInt       v, vx, vy;
393 
394   PetscFunctionBegin;
395   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
396   if (markerSeparate) {
397     markerTop    = 3;
398     markerBottom = 1;
399     markerRight  = 2;
400     markerLeft   = 4;
401   }
402   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
403   if (rank == 0) {
404     PetscInt e, ex, ey;
405 
406     PetscCall(DMPlexSetChart(dm, 0, numEdges + numVertices));
407     for (e = 0; e < numEdges; ++e) { PetscCall(DMPlexSetConeSize(dm, e, 2)); }
408     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
409     for (vx = 0; vx <= edges[0]; vx++) {
410       for (ey = 0; ey < edges[1]; ey++) {
411         PetscInt edge   = vx * edges[1] + ey + edges[0] * (edges[1] + 1);
412         PetscInt vertex = ey * (edges[0] + 1) + vx + numEdges;
413         PetscInt cone[2];
414 
415         cone[0] = vertex;
416         cone[1] = vertex + edges[0] + 1;
417         PetscCall(DMPlexSetCone(dm, edge, cone));
418         if (vx == edges[0]) {
419           PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
420           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
421           if (ey == edges[1] - 1) {
422             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
423             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerRight));
424           }
425         } else if (vx == 0) {
426           PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
427           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
428           if (ey == edges[1] - 1) {
429             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
430             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft));
431           }
432         }
433       }
434     }
435     for (vy = 0; vy <= edges[1]; vy++) {
436       for (ex = 0; ex < edges[0]; ex++) {
437         PetscInt edge   = vy * edges[0] + ex;
438         PetscInt vertex = vy * (edges[0] + 1) + ex + numEdges;
439         PetscInt cone[2];
440 
441         cone[0] = vertex;
442         cone[1] = vertex + 1;
443         PetscCall(DMPlexSetCone(dm, edge, cone));
444         if (vy == edges[1]) {
445           PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
446           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
447           if (ex == edges[0] - 1) {
448             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
449             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerTop));
450           }
451         } else if (vy == 0) {
452           PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
453           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
454           if (ex == edges[0] - 1) {
455             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
456             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom));
457           }
458         }
459       }
460     }
461   }
462   PetscCall(DMPlexSymmetrize(dm));
463   PetscCall(DMPlexStratify(dm));
464   /* Build coordinates */
465   PetscCall(DMSetCoordinateDim(dm, 2));
466   PetscCall(DMGetCoordinateSection(dm, &coordSection));
467   PetscCall(PetscSectionSetNumFields(coordSection, 1));
468   PetscCall(PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices));
469   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 2));
470   for (v = numEdges; v < numEdges + numVertices; ++v) {
471     PetscCall(PetscSectionSetDof(coordSection, v, 2));
472     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 2));
473   }
474   PetscCall(PetscSectionSetUp(coordSection));
475   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
476   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
477   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
478   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
479   PetscCall(VecSetBlockSize(coordinates, 2));
480   PetscCall(VecSetType(coordinates, VECSTANDARD));
481   PetscCall(VecGetArray(coordinates, &coords));
482   for (vy = 0; vy <= edges[1]; ++vy) {
483     for (vx = 0; vx <= edges[0]; ++vx) {
484       coords[(vy * (edges[0] + 1) + vx) * 2 + 0] = lower[0] + ((upper[0] - lower[0]) / edges[0]) * vx;
485       coords[(vy * (edges[0] + 1) + vx) * 2 + 1] = lower[1] + ((upper[1] - lower[1]) / edges[1]) * vy;
486     }
487   }
488   PetscCall(VecRestoreArray(coordinates, &coords));
489   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
490   PetscCall(VecDestroy(&coordinates));
491   PetscFunctionReturn(0);
492 }
493 
494 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[]) {
495   PetscInt     vertices[3], numVertices;
496   PetscInt     numFaces       = 2 * faces[0] * faces[1] + 2 * faces[1] * faces[2] + 2 * faces[0] * faces[2];
497   PetscInt     markerTop      = 1;
498   PetscInt     markerBottom   = 1;
499   PetscInt     markerFront    = 1;
500   PetscInt     markerBack     = 1;
501   PetscInt     markerRight    = 1;
502   PetscInt     markerLeft     = 1;
503   PetscBool    markerSeparate = PETSC_FALSE;
504   Vec          coordinates;
505   PetscSection coordSection;
506   PetscScalar *coords;
507   PetscInt     coordSize;
508   PetscMPIInt  rank;
509   PetscInt     v, vx, vy, vz;
510   PetscInt     voffset, iface = 0, cone[4];
511 
512   PetscFunctionBegin;
513   PetscCheck(faces[0] >= 1 && faces[1] >= 1 && faces[2] >= 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
514   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
515   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
516   if (markerSeparate) {
517     markerBottom = 1;
518     markerTop    = 2;
519     markerFront  = 3;
520     markerBack   = 4;
521     markerRight  = 5;
522     markerLeft   = 6;
523   }
524   vertices[0] = faces[0] + 1;
525   vertices[1] = faces[1] + 1;
526   vertices[2] = faces[2] + 1;
527   numVertices = vertices[0] * vertices[1] * vertices[2];
528   if (rank == 0) {
529     PetscInt f;
530 
531     PetscCall(DMPlexSetChart(dm, 0, numFaces + numVertices));
532     for (f = 0; f < numFaces; ++f) { PetscCall(DMPlexSetConeSize(dm, f, 4)); }
533     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
534 
535     /* Side 0 (Top) */
536     for (vy = 0; vy < faces[1]; vy++) {
537       for (vx = 0; vx < faces[0]; vx++) {
538         voffset = numFaces + vertices[0] * vertices[1] * (vertices[2] - 1) + vy * vertices[0] + vx;
539         cone[0] = voffset;
540         cone[1] = voffset + 1;
541         cone[2] = voffset + vertices[0] + 1;
542         cone[3] = voffset + vertices[0];
543         PetscCall(DMPlexSetCone(dm, iface, cone));
544         PetscCall(DMSetLabelValue(dm, "marker", iface, markerTop));
545         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerTop));
546         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerTop));
547         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerTop));
548         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerTop));
549         iface++;
550       }
551     }
552 
553     /* Side 1 (Bottom) */
554     for (vy = 0; vy < faces[1]; vy++) {
555       for (vx = 0; vx < faces[0]; vx++) {
556         voffset = numFaces + vy * (faces[0] + 1) + vx;
557         cone[0] = voffset + 1;
558         cone[1] = voffset;
559         cone[2] = voffset + vertices[0];
560         cone[3] = voffset + vertices[0] + 1;
561         PetscCall(DMPlexSetCone(dm, iface, cone));
562         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBottom));
563         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBottom));
564         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBottom));
565         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerBottom));
566         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerBottom));
567         iface++;
568       }
569     }
570 
571     /* Side 2 (Front) */
572     for (vz = 0; vz < faces[2]; vz++) {
573       for (vx = 0; vx < faces[0]; vx++) {
574         voffset = numFaces + vz * vertices[0] * vertices[1] + vx;
575         cone[0] = voffset;
576         cone[1] = voffset + 1;
577         cone[2] = voffset + vertices[0] * vertices[1] + 1;
578         cone[3] = voffset + vertices[0] * vertices[1];
579         PetscCall(DMPlexSetCone(dm, iface, cone));
580         PetscCall(DMSetLabelValue(dm, "marker", iface, markerFront));
581         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerFront));
582         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerFront));
583         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerFront));
584         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerFront));
585         iface++;
586       }
587     }
588 
589     /* Side 3 (Back) */
590     for (vz = 0; vz < faces[2]; vz++) {
591       for (vx = 0; vx < faces[0]; vx++) {
592         voffset = numFaces + vz * vertices[0] * vertices[1] + vertices[0] * (vertices[1] - 1) + vx;
593         cone[0] = voffset + vertices[0] * vertices[1];
594         cone[1] = voffset + vertices[0] * vertices[1] + 1;
595         cone[2] = voffset + 1;
596         cone[3] = voffset;
597         PetscCall(DMPlexSetCone(dm, iface, cone));
598         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBack));
599         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBack));
600         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBack));
601         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerBack));
602         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerBack));
603         iface++;
604       }
605     }
606 
607     /* Side 4 (Left) */
608     for (vz = 0; vz < faces[2]; vz++) {
609       for (vy = 0; vy < faces[1]; vy++) {
610         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0];
611         cone[0] = voffset;
612         cone[1] = voffset + vertices[0] * vertices[1];
613         cone[2] = voffset + vertices[0] * vertices[1] + vertices[0];
614         cone[3] = voffset + vertices[0];
615         PetscCall(DMPlexSetCone(dm, iface, cone));
616         PetscCall(DMSetLabelValue(dm, "marker", iface, markerLeft));
617         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerLeft));
618         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerLeft));
619         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[1] + 0, markerLeft));
620         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerLeft));
621         iface++;
622       }
623     }
624 
625     /* Side 5 (Right) */
626     for (vz = 0; vz < faces[2]; vz++) {
627       for (vy = 0; vy < faces[1]; vy++) {
628         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0] + faces[0];
629         cone[0] = voffset + vertices[0] * vertices[1];
630         cone[1] = voffset;
631         cone[2] = voffset + vertices[0];
632         cone[3] = voffset + vertices[0] * vertices[1] + vertices[0];
633         PetscCall(DMPlexSetCone(dm, iface, cone));
634         PetscCall(DMSetLabelValue(dm, "marker", iface, markerRight));
635         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerRight));
636         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerRight));
637         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerRight));
638         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerRight));
639         iface++;
640       }
641     }
642   }
643   PetscCall(DMPlexSymmetrize(dm));
644   PetscCall(DMPlexStratify(dm));
645   /* Build coordinates */
646   PetscCall(DMSetCoordinateDim(dm, 3));
647   PetscCall(DMGetCoordinateSection(dm, &coordSection));
648   PetscCall(PetscSectionSetNumFields(coordSection, 1));
649   PetscCall(PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices));
650   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 3));
651   for (v = numFaces; v < numFaces + numVertices; ++v) {
652     PetscCall(PetscSectionSetDof(coordSection, v, 3));
653     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 3));
654   }
655   PetscCall(PetscSectionSetUp(coordSection));
656   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
657   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
658   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
659   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
660   PetscCall(VecSetBlockSize(coordinates, 3));
661   PetscCall(VecSetType(coordinates, VECSTANDARD));
662   PetscCall(VecGetArray(coordinates, &coords));
663   for (vz = 0; vz <= faces[2]; ++vz) {
664     for (vy = 0; vy <= faces[1]; ++vy) {
665       for (vx = 0; vx <= faces[0]; ++vx) {
666         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 0] = lower[0] + ((upper[0] - lower[0]) / faces[0]) * vx;
667         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 1] = lower[1] + ((upper[1] - lower[1]) / faces[1]) * vy;
668         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 2] = lower[2] + ((upper[2] - lower[2]) / faces[2]) * vz;
669       }
670     }
671   }
672   PetscCall(VecRestoreArray(coordinates, &coords));
673   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
674   PetscCall(VecDestroy(&coordinates));
675   PetscFunctionReturn(0);
676 }
677 
678 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate) {
679   PetscFunctionBegin;
680   PetscValidLogicalCollectiveInt(dm, dim, 2);
681   PetscCall(DMSetDimension(dm, dim - 1));
682   PetscCall(DMSetCoordinateDim(dm, dim));
683   switch (dim) {
684   case 1: PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(dm, lower, upper, faces)); break;
685   case 2: PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(dm, lower, upper, faces)); break;
686   case 3: PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(dm, lower, upper, faces)); break;
687   default: SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Dimension not supported: %" PetscInt_FMT, dim);
688   }
689   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
690   PetscFunctionReturn(0);
691 }
692 
693 /*@C
694   DMPlexCreateBoxSurfaceMesh - Creates a mesh on the surface of the tensor product of unit intervals (box) using tensor cells (hexahedra).
695 
696   Collective
697 
698   Input Parameters:
699 + comm        - The communicator for the DM object
700 . dim         - The spatial dimension of the box, so the resulting mesh is has dimension dim-1
701 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
702 . lower       - The lower left corner, or NULL for (0, 0, 0)
703 . upper       - The upper right corner, or NULL for (1, 1, 1)
704 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
705 
706   Output Parameter:
707 . dm  - The DM object
708 
709   Level: beginner
710 
711 .seealso: `DMSetFromOptions()`, `DMPlexCreateBoxMesh()`, `DMPlexCreateFromFile()`, `DMSetType()`, `DMCreate()`
712 @*/
713 PetscErrorCode DMPlexCreateBoxSurfaceMesh(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate, DM *dm) {
714   PetscInt  fac[3] = {1, 1, 1};
715   PetscReal low[3] = {0, 0, 0};
716   PetscReal upp[3] = {1, 1, 1};
717 
718   PetscFunctionBegin;
719   PetscCall(DMCreate(comm, dm));
720   PetscCall(DMSetType(*dm, DMPLEX));
721   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(*dm, dim, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, interpolate));
722   PetscFunctionReturn(0);
723 }
724 
725 static PetscErrorCode DMPlexCreateLineMesh_Internal(DM dm, PetscInt segments, PetscReal lower, PetscReal upper, DMBoundaryType bd) {
726   PetscInt     i, fStart, fEnd, numCells = 0, numVerts = 0;
727   PetscInt     numPoints[2], *coneSize, *cones, *coneOrientations;
728   PetscScalar *vertexCoords;
729   PetscReal    L, maxCell;
730   PetscBool    markerSeparate = PETSC_FALSE;
731   PetscInt     markerLeft = 1, faceMarkerLeft = 1;
732   PetscInt     markerRight = 1, faceMarkerRight = 2;
733   PetscBool    wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
734   PetscMPIInt  rank;
735 
736   PetscFunctionBegin;
737   PetscValidPointer(dm, 1);
738 
739   PetscCall(DMSetDimension(dm, 1));
740   PetscCall(DMCreateLabel(dm, "marker"));
741   PetscCall(DMCreateLabel(dm, "Face Sets"));
742 
743   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
744   if (rank == 0) numCells = segments;
745   if (rank == 0) numVerts = segments + (wrap ? 0 : 1);
746 
747   numPoints[0] = numVerts;
748   numPoints[1] = numCells;
749   PetscCall(PetscMalloc4(numCells + numVerts, &coneSize, numCells * 2, &cones, numCells + numVerts, &coneOrientations, numVerts, &vertexCoords));
750   PetscCall(PetscArrayzero(coneOrientations, numCells + numVerts));
751   for (i = 0; i < numCells; ++i) { coneSize[i] = 2; }
752   for (i = 0; i < numVerts; ++i) { coneSize[numCells + i] = 0; }
753   for (i = 0; i < numCells; ++i) {
754     cones[2 * i]     = numCells + i % numVerts;
755     cones[2 * i + 1] = numCells + (i + 1) % numVerts;
756   }
757   for (i = 0; i < numVerts; ++i) { vertexCoords[i] = lower + (upper - lower) * ((PetscReal)i / (PetscReal)numCells); }
758   PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
759   PetscCall(PetscFree4(coneSize, cones, coneOrientations, vertexCoords));
760 
761   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
762   if (markerSeparate) {
763     markerLeft  = faceMarkerLeft;
764     markerRight = faceMarkerRight;
765   }
766   if (!wrap && rank == 0) {
767     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
768     PetscCall(DMSetLabelValue(dm, "marker", fStart, markerLeft));
769     PetscCall(DMSetLabelValue(dm, "marker", fEnd - 1, markerRight));
770     PetscCall(DMSetLabelValue(dm, "Face Sets", fStart, faceMarkerLeft));
771     PetscCall(DMSetLabelValue(dm, "Face Sets", fEnd - 1, faceMarkerRight));
772   }
773   if (wrap) {
774     L       = upper - lower;
775     maxCell = (PetscReal)1.1 * (L / (PetscReal)PetscMax(1, segments));
776     PetscCall(DMSetPeriodicity(dm, &maxCell, &lower, &L));
777   }
778   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
779   PetscFunctionReturn(0);
780 }
781 
782 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate) {
783   DM      boundary, vol;
784   DMLabel bdlabel;
785 
786   PetscFunctionBegin;
787   PetscValidPointer(dm, 1);
788   for (PetscInt i = 0; i < dim; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
789   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &boundary));
790   PetscCall(DMSetType(boundary, DMPLEX));
791   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(boundary, dim, faces, lower, upper, PETSC_FALSE));
792   PetscCall(DMPlexGenerate(boundary, NULL, interpolate, &vol));
793   PetscCall(DMGetLabel(vol, "marker", &bdlabel));
794   if (bdlabel) PetscCall(DMPlexLabelComplete(vol, bdlabel));
795   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_FALSE, vol));
796   PetscCall(DMPlexReplace_Static(dm, &vol));
797   PetscCall(DMDestroy(&boundary));
798   PetscFunctionReturn(0);
799 }
800 
801 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ) {
802   DMLabel     cutLabel  = NULL;
803   PetscInt    markerTop = 1, faceMarkerTop = 1;
804   PetscInt    markerBottom = 1, faceMarkerBottom = 1;
805   PetscInt    markerFront = 1, faceMarkerFront = 1;
806   PetscInt    markerBack = 1, faceMarkerBack = 1;
807   PetscInt    markerRight = 1, faceMarkerRight = 1;
808   PetscInt    markerLeft = 1, faceMarkerLeft = 1;
809   PetscInt    dim;
810   PetscBool   markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
811   PetscMPIInt rank;
812 
813   PetscFunctionBegin;
814   PetscCall(DMGetDimension(dm, &dim));
815   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
816   PetscCall(DMCreateLabel(dm, "marker"));
817   PetscCall(DMCreateLabel(dm, "Face Sets"));
818   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
819   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST || bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST || bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
820     if (cutMarker) {
821       PetscCall(DMCreateLabel(dm, "periodic_cut"));
822       PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
823     }
824   }
825   switch (dim) {
826   case 2:
827     faceMarkerTop    = 3;
828     faceMarkerBottom = 1;
829     faceMarkerRight  = 2;
830     faceMarkerLeft   = 4;
831     break;
832   case 3:
833     faceMarkerBottom = 1;
834     faceMarkerTop    = 2;
835     faceMarkerFront  = 3;
836     faceMarkerBack   = 4;
837     faceMarkerRight  = 5;
838     faceMarkerLeft   = 6;
839     break;
840   default: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
841   }
842   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
843   if (markerSeparate) {
844     markerBottom = faceMarkerBottom;
845     markerTop    = faceMarkerTop;
846     markerFront  = faceMarkerFront;
847     markerBack   = faceMarkerBack;
848     markerRight  = faceMarkerRight;
849     markerLeft   = faceMarkerLeft;
850   }
851   {
852     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
853     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
854     const PetscInt numZEdges    = rank == 0 ? edges[2] : 0;
855     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0] + 1) : 0;
856     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1] + 1) : 0;
857     const PetscInt numZVertices = rank == 0 ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2] + 1) : 0;
858     const PetscInt numCells     = numXEdges * numYEdges * numZEdges;
859     const PetscInt numXFaces    = numYEdges * numZEdges;
860     const PetscInt numYFaces    = numXEdges * numZEdges;
861     const PetscInt numZFaces    = numXEdges * numYEdges;
862     const PetscInt numTotXFaces = numXVertices * numXFaces;
863     const PetscInt numTotYFaces = numYVertices * numYFaces;
864     const PetscInt numTotZFaces = numZVertices * numZFaces;
865     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
866     const PetscInt numTotXEdges = numXEdges * numYVertices * numZVertices;
867     const PetscInt numTotYEdges = numYEdges * numXVertices * numZVertices;
868     const PetscInt numTotZEdges = numZEdges * numXVertices * numYVertices;
869     const PetscInt numVertices  = numXVertices * numYVertices * numZVertices;
870     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
871     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
872     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
873     const PetscInt firstYFace   = firstXFace + numTotXFaces;
874     const PetscInt firstZFace   = firstYFace + numTotYFaces;
875     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
876     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
877     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
878     Vec            coordinates;
879     PetscSection   coordSection;
880     PetscScalar   *coords;
881     PetscInt       coordSize;
882     PetscInt       v, vx, vy, vz;
883     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
884 
885     PetscCall(DMPlexSetChart(dm, 0, numCells + numFaces + numEdges + numVertices));
886     for (c = 0; c < numCells; c++) { PetscCall(DMPlexSetConeSize(dm, c, 6)); }
887     for (f = firstXFace; f < firstXFace + numFaces; ++f) { PetscCall(DMPlexSetConeSize(dm, f, 4)); }
888     for (e = firstXEdge; e < firstXEdge + numEdges; ++e) { PetscCall(DMPlexSetConeSize(dm, e, 2)); }
889     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
890     /* Build cells */
891     for (fz = 0; fz < numZEdges; ++fz) {
892       for (fy = 0; fy < numYEdges; ++fy) {
893         for (fx = 0; fx < numXEdges; ++fx) {
894           PetscInt cell    = (fz * numYEdges + fy) * numXEdges + fx;
895           PetscInt faceB   = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
896           PetscInt faceT   = firstZFace + (fy * numXEdges + fx) * numZVertices + ((fz + 1) % numZVertices);
897           PetscInt faceF   = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
898           PetscInt faceK   = firstYFace + (fz * numXEdges + fx) * numYVertices + ((fy + 1) % numYVertices);
899           PetscInt faceL   = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
900           PetscInt faceR   = firstXFace + (fz * numYEdges + fy) * numXVertices + ((fx + 1) % numXVertices);
901           /* B,  T,  F,  K,  R,  L */
902           PetscInt ornt[6] = {-2, 0, 0, -3, 0, -2}; /* ??? */
903           PetscInt cone[6];
904 
905           /* no boundary twisting in 3D */
906           cone[0] = faceB;
907           cone[1] = faceT;
908           cone[2] = faceF;
909           cone[3] = faceK;
910           cone[4] = faceR;
911           cone[5] = faceL;
912           PetscCall(DMPlexSetCone(dm, cell, cone));
913           PetscCall(DMPlexSetConeOrientation(dm, cell, ornt));
914           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
915           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
916           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
917         }
918       }
919     }
920     /* Build x faces */
921     for (fz = 0; fz < numZEdges; ++fz) {
922       for (fy = 0; fy < numYEdges; ++fy) {
923         for (fx = 0; fx < numXVertices; ++fx) {
924           PetscInt face    = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
925           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
926           PetscInt edgeR   = firstZEdge + (((fy + 1) % numYVertices) * numXVertices + fx) * numZEdges + fz;
927           PetscInt edgeB   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
928           PetscInt edgeT   = firstYEdge + (((fz + 1) % numZVertices) * numXVertices + fx) * numYEdges + fy;
929           PetscInt ornt[4] = {0, 0, -1, -1};
930           PetscInt cone[4];
931 
932           if (dim == 3) {
933             /* markers */
934             if (bdX != DM_BOUNDARY_PERIODIC) {
935               if (fx == numXVertices - 1) {
936                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight));
937                 PetscCall(DMSetLabelValue(dm, "marker", face, markerRight));
938               } else if (fx == 0) {
939                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft));
940                 PetscCall(DMSetLabelValue(dm, "marker", face, markerLeft));
941               }
942             }
943           }
944           cone[0] = edgeB;
945           cone[1] = edgeR;
946           cone[2] = edgeT;
947           cone[3] = edgeL;
948           PetscCall(DMPlexSetCone(dm, face, cone));
949           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
950         }
951       }
952     }
953     /* Build y faces */
954     for (fz = 0; fz < numZEdges; ++fz) {
955       for (fx = 0; fx < numXEdges; ++fx) {
956         for (fy = 0; fy < numYVertices; ++fy) {
957           PetscInt face    = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
958           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
959           PetscInt edgeR   = firstZEdge + (fy * numXVertices + ((fx + 1) % numXVertices)) * numZEdges + fz;
960           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
961           PetscInt edgeT   = firstXEdge + (((fz + 1) % numZVertices) * numYVertices + fy) * numXEdges + fx;
962           PetscInt ornt[4] = {0, 0, -1, -1};
963           PetscInt cone[4];
964 
965           if (dim == 3) {
966             /* markers */
967             if (bdY != DM_BOUNDARY_PERIODIC) {
968               if (fy == numYVertices - 1) {
969                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack));
970                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBack));
971               } else if (fy == 0) {
972                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront));
973                 PetscCall(DMSetLabelValue(dm, "marker", face, markerFront));
974               }
975             }
976           }
977           cone[0] = edgeB;
978           cone[1] = edgeR;
979           cone[2] = edgeT;
980           cone[3] = edgeL;
981           PetscCall(DMPlexSetCone(dm, face, cone));
982           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
983         }
984       }
985     }
986     /* Build z faces */
987     for (fy = 0; fy < numYEdges; ++fy) {
988       for (fx = 0; fx < numXEdges; ++fx) {
989         for (fz = 0; fz < numZVertices; fz++) {
990           PetscInt face    = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
991           PetscInt edgeL   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
992           PetscInt edgeR   = firstYEdge + (fz * numXVertices + ((fx + 1) % numXVertices)) * numYEdges + fy;
993           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
994           PetscInt edgeT   = firstXEdge + (fz * numYVertices + ((fy + 1) % numYVertices)) * numXEdges + fx;
995           PetscInt ornt[4] = {0, 0, -1, -1};
996           PetscInt cone[4];
997 
998           if (dim == 2) {
999             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges - 1) {
1000               edgeR += numYEdges - 1 - 2 * fy;
1001               ornt[1] = -1;
1002             }
1003             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges - 1) {
1004               edgeT += numXEdges - 1 - 2 * fx;
1005               ornt[2] = 0;
1006             }
1007             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1008             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1009           } else {
1010             /* markers */
1011             if (bdZ != DM_BOUNDARY_PERIODIC) {
1012               if (fz == numZVertices - 1) {
1013                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop));
1014                 PetscCall(DMSetLabelValue(dm, "marker", face, markerTop));
1015               } else if (fz == 0) {
1016                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom));
1017                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBottom));
1018               }
1019             }
1020           }
1021           cone[0] = edgeB;
1022           cone[1] = edgeR;
1023           cone[2] = edgeT;
1024           cone[3] = edgeL;
1025           PetscCall(DMPlexSetCone(dm, face, cone));
1026           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1027         }
1028       }
1029     }
1030     /* Build Z edges*/
1031     for (vy = 0; vy < numYVertices; vy++) {
1032       for (vx = 0; vx < numXVertices; vx++) {
1033         for (ez = 0; ez < numZEdges; ez++) {
1034           const PetscInt edge    = firstZEdge + (vy * numXVertices + vx) * numZEdges + ez;
1035           const PetscInt vertexB = firstVertex + (ez * numYVertices + vy) * numXVertices + vx;
1036           const PetscInt vertexT = firstVertex + (((ez + 1) % numZVertices) * numYVertices + vy) * numXVertices + vx;
1037           PetscInt       cone[2];
1038 
1039           cone[0] = vertexB;
1040           cone[1] = vertexT;
1041           PetscCall(DMPlexSetCone(dm, edge, cone));
1042           if (dim == 3) {
1043             if (bdX != DM_BOUNDARY_PERIODIC) {
1044               if (vx == numXVertices - 1) {
1045                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1046                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1047                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1048               } else if (vx == 0) {
1049                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1050                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1051                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1052               }
1053             }
1054             if (bdY != DM_BOUNDARY_PERIODIC) {
1055               if (vy == numYVertices - 1) {
1056                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1057                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1058                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1059               } else if (vy == 0) {
1060                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1061                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1062                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1063               }
1064             }
1065           }
1066         }
1067       }
1068     }
1069     /* Build Y edges*/
1070     for (vz = 0; vz < numZVertices; vz++) {
1071       for (vx = 0; vx < numXVertices; vx++) {
1072         for (ey = 0; ey < numYEdges; ey++) {
1073           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges - 1) ? (numXVertices - vx - 1) : (vz * numYVertices + ((ey + 1) % numYVertices)) * numXVertices + vx;
1074           const PetscInt edge    = firstYEdge + (vz * numXVertices + vx) * numYEdges + ey;
1075           const PetscInt vertexF = firstVertex + (vz * numYVertices + ey) * numXVertices + vx;
1076           const PetscInt vertexK = firstVertex + nextv;
1077           PetscInt       cone[2];
1078 
1079           cone[0] = vertexF;
1080           cone[1] = vertexK;
1081           PetscCall(DMPlexSetCone(dm, edge, cone));
1082           if (dim == 2) {
1083             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1084               if (vx == numXVertices - 1) {
1085                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight));
1086                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1087                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1088                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1089               } else if (vx == 0) {
1090                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft));
1091                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1092                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1093                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1094               }
1095             } else {
1096               if (vx == 0 && cutLabel) {
1097                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1098                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1099                 if (ey == numYEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1100               }
1101             }
1102           } else {
1103             if (bdX != DM_BOUNDARY_PERIODIC) {
1104               if (vx == numXVertices - 1) {
1105                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1106                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1107                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1108               } else if (vx == 0) {
1109                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1110                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1111                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1112               }
1113             }
1114             if (bdZ != DM_BOUNDARY_PERIODIC) {
1115               if (vz == numZVertices - 1) {
1116                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1117                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1118                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1119               } else if (vz == 0) {
1120                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1121                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1122                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1123               }
1124             }
1125           }
1126         }
1127       }
1128     }
1129     /* Build X edges*/
1130     for (vz = 0; vz < numZVertices; vz++) {
1131       for (vy = 0; vy < numYVertices; vy++) {
1132         for (ex = 0; ex < numXEdges; ex++) {
1133           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges - 1) ? (numYVertices - vy - 1) * numXVertices : (vz * numYVertices + vy) * numXVertices + (ex + 1) % numXVertices;
1134           const PetscInt edge    = firstXEdge + (vz * numYVertices + vy) * numXEdges + ex;
1135           const PetscInt vertexL = firstVertex + (vz * numYVertices + vy) * numXVertices + ex;
1136           const PetscInt vertexR = firstVertex + nextv;
1137           PetscInt       cone[2];
1138 
1139           cone[0] = vertexL;
1140           cone[1] = vertexR;
1141           PetscCall(DMPlexSetCone(dm, edge, cone));
1142           if (dim == 2) {
1143             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1144               if (vy == numYVertices - 1) {
1145                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop));
1146                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1147                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1148                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1149               } else if (vy == 0) {
1150                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom));
1151                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1152                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1153                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1154               }
1155             } else {
1156               if (vy == 0 && cutLabel) {
1157                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1158                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1159                 if (ex == numXEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1160               }
1161             }
1162           } else {
1163             if (bdY != DM_BOUNDARY_PERIODIC) {
1164               if (vy == numYVertices - 1) {
1165                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1166                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1167                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1168               } else if (vy == 0) {
1169                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1170                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1171                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1172               }
1173             }
1174             if (bdZ != DM_BOUNDARY_PERIODIC) {
1175               if (vz == numZVertices - 1) {
1176                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1177                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1178                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1179               } else if (vz == 0) {
1180                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1181                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1182                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1183               }
1184             }
1185           }
1186         }
1187       }
1188     }
1189     PetscCall(DMPlexSymmetrize(dm));
1190     PetscCall(DMPlexStratify(dm));
1191     /* Build coordinates */
1192     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1193     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1194     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1195     PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVertices));
1196     for (v = firstVertex; v < firstVertex + numVertices; ++v) {
1197       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1198       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1199     }
1200     PetscCall(PetscSectionSetUp(coordSection));
1201     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1202     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1203     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1204     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1205     PetscCall(VecSetBlockSize(coordinates, dim));
1206     PetscCall(VecSetType(coordinates, VECSTANDARD));
1207     PetscCall(VecGetArray(coordinates, &coords));
1208     for (vz = 0; vz < numZVertices; ++vz) {
1209       for (vy = 0; vy < numYVertices; ++vy) {
1210         for (vx = 0; vx < numXVertices; ++vx) {
1211           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * vx;
1212           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * vy;
1213           if (dim == 3) { coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 2] = lower[2] + ((upper[2] - lower[2]) / numZEdges) * vz; }
1214         }
1215       }
1216     }
1217     PetscCall(VecRestoreArray(coordinates, &coords));
1218     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1219     PetscCall(VecDestroy(&coordinates));
1220   }
1221   PetscFunctionReturn(0);
1222 }
1223 
1224 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[]) {
1225   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1226   PetscInt       fac[3] = {0, 0, 0}, d;
1227 
1228   PetscFunctionBegin;
1229   PetscValidPointer(dm, 1);
1230   PetscValidLogicalCollectiveInt(dm, dim, 2);
1231   PetscCall(DMSetDimension(dm, dim));
1232   for (d = 0; d < dim; ++d) {
1233     fac[d] = faces[d];
1234     bdt[d] = periodicity[d];
1235   }
1236   PetscCall(DMPlexCreateCubeMesh_Internal(dm, lower, upper, fac, bdt[0], bdt[1], bdt[2]));
1237   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST || periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST || (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
1238     PetscReal L[3]       = {-1., -1., 0.};
1239     PetscReal maxCell[3] = {-1., -1., 0.};
1240 
1241     for (d = 0; d < dim; ++d) {
1242       if (periodicity[d] != DM_BOUNDARY_NONE) {
1243         L[d]       = upper[d] - lower[d];
1244         maxCell[d] = 1.1 * (L[d] / PetscMax(1, faces[d]));
1245       }
1246     }
1247     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1248   }
1249   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
1250   PetscFunctionReturn(0);
1251 }
1252 
1253 static PetscErrorCode DMPlexCreateBoxMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate) {
1254   PetscFunctionBegin;
1255   if (dim == 1) PetscCall(DMPlexCreateLineMesh_Internal(dm, faces[0], lower[0], upper[0], periodicity[0]));
1256   else if (simplex) PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1257   else PetscCall(DMPlexCreateBoxMesh_Tensor_Internal(dm, dim, faces, lower, upper, periodicity));
1258   if (!interpolate && dim > 1 && !simplex) {
1259     DM udm;
1260 
1261     PetscCall(DMPlexUninterpolate(dm, &udm));
1262     PetscCall(DMPlexCopyCoordinates(dm, udm));
1263     PetscCall(DMPlexReplace_Static(dm, &udm));
1264   }
1265   PetscFunctionReturn(0);
1266 }
1267 
1268 /*@C
1269   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
1270 
1271   Collective
1272 
1273   Input Parameters:
1274 + comm        - The communicator for the DM object
1275 . dim         - The spatial dimension
1276 . simplex     - PETSC_TRUE for simplices, PETSC_FALSE for tensor cells
1277 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1278 . lower       - The lower left corner, or NULL for (0, 0, 0)
1279 . upper       - The upper right corner, or NULL for (1, 1, 1)
1280 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
1281 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1282 
1283   Output Parameter:
1284 . dm  - The DM object
1285 
1286   Note: If you want to customize this mesh using options, you just need to
1287 $  DMCreate(comm, &dm);
1288 $  DMSetType(dm, DMPLEX);
1289 $  DMSetFromOptions(dm);
1290 and use the options on the DMSetFromOptions() page.
1291 
1292   Here is the numbering returned for 2 faces in each direction for tensor cells:
1293 $ 10---17---11---18----12
1294 $  |         |         |
1295 $  |         |         |
1296 $ 20    2   22    3    24
1297 $  |         |         |
1298 $  |         |         |
1299 $  7---15----8---16----9
1300 $  |         |         |
1301 $  |         |         |
1302 $ 19    0   21    1   23
1303 $  |         |         |
1304 $  |         |         |
1305 $  4---13----5---14----6
1306 
1307 and for simplicial cells
1308 
1309 $ 14----8---15----9----16
1310 $  |\     5  |\      7 |
1311 $  | \       | \       |
1312 $ 13   2    14    3    15
1313 $  | 4   \   | 6   \   |
1314 $  |       \ |       \ |
1315 $ 11----6---12----7----13
1316 $  |\        |\        |
1317 $  | \    1  | \     3 |
1318 $ 10   0    11    1    12
1319 $  | 0   \   | 2   \   |
1320 $  |       \ |       \ |
1321 $  8----4----9----5----10
1322 
1323   Level: beginner
1324 
1325 .seealso: `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
1326 @*/
1327 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm) {
1328   PetscInt       fac[3] = {1, 1, 1};
1329   PetscReal      low[3] = {0, 0, 0};
1330   PetscReal      upp[3] = {1, 1, 1};
1331   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1332 
1333   PetscFunctionBegin;
1334   PetscCall(DMCreate(comm, dm));
1335   PetscCall(DMSetType(*dm, DMPLEX));
1336   PetscCall(DMPlexCreateBoxMesh_Internal(*dm, dim, simplex, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt, interpolate));
1337   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1338   PetscFunctionReturn(0);
1339 }
1340 
1341 static PetscErrorCode DMPlexCreateWedgeBoxMesh_Internal(DM dm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[]) {
1342   DM       bdm, vol;
1343   PetscInt i;
1344 
1345   PetscFunctionBegin;
1346   for (i = 0; i < 3; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity not yet supported");
1347   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &bdm));
1348   PetscCall(DMSetType(bdm, DMPLEX));
1349   PetscCall(DMSetDimension(bdm, 2));
1350   PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(bdm, 2, faces, lower, upper, periodicity, PETSC_TRUE));
1351   PetscCall(DMPlexExtrude(bdm, faces[2], upper[2] - lower[2], PETSC_TRUE, PETSC_FALSE, NULL, NULL, &vol));
1352   PetscCall(DMDestroy(&bdm));
1353   PetscCall(DMPlexReplace_Static(dm, &vol));
1354   if (lower[2] != 0.0) {
1355     Vec          v;
1356     PetscScalar *x;
1357     PetscInt     cDim, n;
1358 
1359     PetscCall(DMGetCoordinatesLocal(dm, &v));
1360     PetscCall(VecGetBlockSize(v, &cDim));
1361     PetscCall(VecGetLocalSize(v, &n));
1362     PetscCall(VecGetArray(v, &x));
1363     x += cDim;
1364     for (i = 0; i < n; i += cDim) x[i] += lower[2];
1365     PetscCall(VecRestoreArray(v, &x));
1366     PetscCall(DMSetCoordinatesLocal(dm, v));
1367   }
1368   PetscFunctionReturn(0);
1369 }
1370 
1371 /*@
1372   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tesselating the (x,y) plane and extruding in the third direction using wedge cells.
1373 
1374   Collective
1375 
1376   Input Parameters:
1377 + comm        - The communicator for the DM object
1378 . faces       - Number of faces per dimension, or NULL for (1, 1, 1)
1379 . lower       - The lower left corner, or NULL for (0, 0, 0)
1380 . upper       - The upper right corner, or NULL for (1, 1, 1)
1381 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
1382 . orderHeight - If PETSC_TRUE, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1383 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1384 
1385   Output Parameter:
1386 . dm  - The DM object
1387 
1388   Level: beginner
1389 
1390 .seealso: `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateWedgeCylinderMesh()`, `DMExtrude()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1391 @*/
1392 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool orderHeight, PetscBool interpolate, DM *dm) {
1393   PetscInt       fac[3] = {1, 1, 1};
1394   PetscReal      low[3] = {0, 0, 0};
1395   PetscReal      upp[3] = {1, 1, 1};
1396   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1397 
1398   PetscFunctionBegin;
1399   PetscCall(DMCreate(comm, dm));
1400   PetscCall(DMSetType(*dm, DMPLEX));
1401   PetscCall(DMPlexCreateWedgeBoxMesh_Internal(*dm, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt));
1402   if (!interpolate) {
1403     DM udm;
1404 
1405     PetscCall(DMPlexUninterpolate(*dm, &udm));
1406     PetscCall(DMPlexReplace_Static(*dm, &udm));
1407   }
1408   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1409   PetscFunctionReturn(0);
1410 }
1411 
1412 /*@C
1413   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all DM options in the database.
1414 
1415   Logically Collective on dm
1416 
1417   Input Parameters:
1418 + dm - the DM context
1419 - prefix - the prefix to prepend to all option names
1420 
1421   Notes:
1422   A hyphen (-) must NOT be given at the beginning of the prefix name.
1423   The first character of all runtime options is AUTOMATICALLY the hyphen.
1424 
1425   Level: advanced
1426 
1427 .seealso: `SNESSetFromOptions()`
1428 @*/
1429 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[]) {
1430   DM_Plex *mesh = (DM_Plex *)dm->data;
1431 
1432   PetscFunctionBegin;
1433   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1434   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, prefix));
1435   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)mesh->partitioner, prefix));
1436   PetscFunctionReturn(0);
1437 }
1438 
1439 /* Remap geometry to cylinder
1440    TODO: This only works for a single refinement, then it is broken
1441 
1442      Interior square: Linear interpolation is correct
1443      The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
1444      such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
1445 
1446        phi     = arctan(y/x)
1447        d_close = sqrt(1/8 + 1/4 sin^2(phi))
1448        d_far   = sqrt(1/2 + sin^2(phi))
1449 
1450      so we remap them using
1451 
1452        x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
1453        y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
1454 
1455      If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
1456 */
1457 static void snapToCylinder(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[]) {
1458   const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1459   const PetscReal ds2 = 0.5 * dis;
1460 
1461   if ((PetscAbsScalar(u[0]) <= ds2) && (PetscAbsScalar(u[1]) <= ds2)) {
1462     f0[0] = u[0];
1463     f0[1] = u[1];
1464   } else {
1465     PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
1466 
1467     x    = PetscRealPart(u[0]);
1468     y    = PetscRealPart(u[1]);
1469     phi  = PetscAtan2Real(y, x);
1470     sinp = PetscSinReal(phi);
1471     cosp = PetscCosReal(phi);
1472     if ((PetscAbsReal(phi) > PETSC_PI / 4.0) && (PetscAbsReal(phi) < 3.0 * PETSC_PI / 4.0)) {
1473       dc = PetscAbsReal(ds2 / sinp);
1474       df = PetscAbsReal(dis / sinp);
1475       xc = ds2 * x / PetscAbsReal(y);
1476       yc = ds2 * PetscSignReal(y);
1477     } else {
1478       dc = PetscAbsReal(ds2 / cosp);
1479       df = PetscAbsReal(dis / cosp);
1480       xc = ds2 * PetscSignReal(x);
1481       yc = ds2 * y / PetscAbsReal(x);
1482     }
1483     f0[0] = xc + (u[0] - xc) * (1.0 - dc) / (df - dc);
1484     f0[1] = yc + (u[1] - yc) * (1.0 - dc) / (df - dc);
1485   }
1486   f0[2] = u[2];
1487 }
1488 
1489 static PetscErrorCode DMPlexCreateHexCylinderMesh_Internal(DM dm, DMBoundaryType periodicZ) {
1490   const PetscInt dim = 3;
1491   PetscInt       numCells, numVertices;
1492   PetscMPIInt    rank;
1493 
1494   PetscFunctionBegin;
1495   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1496   PetscCall(DMSetDimension(dm, dim));
1497   /* Create topology */
1498   {
1499     PetscInt cone[8], c;
1500 
1501     numCells    = rank == 0 ? 5 : 0;
1502     numVertices = rank == 0 ? 16 : 0;
1503     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1504       numCells *= 3;
1505       numVertices = rank == 0 ? 24 : 0;
1506     }
1507     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1508     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 8));
1509     PetscCall(DMSetUp(dm));
1510     if (rank == 0) {
1511       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1512         cone[0] = 15;
1513         cone[1] = 18;
1514         cone[2] = 17;
1515         cone[3] = 16;
1516         cone[4] = 31;
1517         cone[5] = 32;
1518         cone[6] = 33;
1519         cone[7] = 34;
1520         PetscCall(DMPlexSetCone(dm, 0, cone));
1521         cone[0] = 16;
1522         cone[1] = 17;
1523         cone[2] = 24;
1524         cone[3] = 23;
1525         cone[4] = 32;
1526         cone[5] = 36;
1527         cone[6] = 37;
1528         cone[7] = 33; /* 22 25 26 21 */
1529         PetscCall(DMPlexSetCone(dm, 1, cone));
1530         cone[0] = 18;
1531         cone[1] = 27;
1532         cone[2] = 24;
1533         cone[3] = 17;
1534         cone[4] = 34;
1535         cone[5] = 33;
1536         cone[6] = 37;
1537         cone[7] = 38;
1538         PetscCall(DMPlexSetCone(dm, 2, cone));
1539         cone[0] = 29;
1540         cone[1] = 27;
1541         cone[2] = 18;
1542         cone[3] = 15;
1543         cone[4] = 35;
1544         cone[5] = 31;
1545         cone[6] = 34;
1546         cone[7] = 38;
1547         PetscCall(DMPlexSetCone(dm, 3, cone));
1548         cone[0] = 29;
1549         cone[1] = 15;
1550         cone[2] = 16;
1551         cone[3] = 23;
1552         cone[4] = 35;
1553         cone[5] = 36;
1554         cone[6] = 32;
1555         cone[7] = 31;
1556         PetscCall(DMPlexSetCone(dm, 4, cone));
1557 
1558         cone[0] = 31;
1559         cone[1] = 34;
1560         cone[2] = 33;
1561         cone[3] = 32;
1562         cone[4] = 19;
1563         cone[5] = 22;
1564         cone[6] = 21;
1565         cone[7] = 20;
1566         PetscCall(DMPlexSetCone(dm, 5, cone));
1567         cone[0] = 32;
1568         cone[1] = 33;
1569         cone[2] = 37;
1570         cone[3] = 36;
1571         cone[4] = 22;
1572         cone[5] = 25;
1573         cone[6] = 26;
1574         cone[7] = 21;
1575         PetscCall(DMPlexSetCone(dm, 6, cone));
1576         cone[0] = 34;
1577         cone[1] = 38;
1578         cone[2] = 37;
1579         cone[3] = 33;
1580         cone[4] = 20;
1581         cone[5] = 21;
1582         cone[6] = 26;
1583         cone[7] = 28;
1584         PetscCall(DMPlexSetCone(dm, 7, cone));
1585         cone[0] = 35;
1586         cone[1] = 38;
1587         cone[2] = 34;
1588         cone[3] = 31;
1589         cone[4] = 30;
1590         cone[5] = 19;
1591         cone[6] = 20;
1592         cone[7] = 28;
1593         PetscCall(DMPlexSetCone(dm, 8, cone));
1594         cone[0] = 35;
1595         cone[1] = 31;
1596         cone[2] = 32;
1597         cone[3] = 36;
1598         cone[4] = 30;
1599         cone[5] = 25;
1600         cone[6] = 22;
1601         cone[7] = 19;
1602         PetscCall(DMPlexSetCone(dm, 9, cone));
1603 
1604         cone[0] = 19;
1605         cone[1] = 20;
1606         cone[2] = 21;
1607         cone[3] = 22;
1608         cone[4] = 15;
1609         cone[5] = 16;
1610         cone[6] = 17;
1611         cone[7] = 18;
1612         PetscCall(DMPlexSetCone(dm, 10, cone));
1613         cone[0] = 22;
1614         cone[1] = 21;
1615         cone[2] = 26;
1616         cone[3] = 25;
1617         cone[4] = 16;
1618         cone[5] = 23;
1619         cone[6] = 24;
1620         cone[7] = 17;
1621         PetscCall(DMPlexSetCone(dm, 11, cone));
1622         cone[0] = 20;
1623         cone[1] = 28;
1624         cone[2] = 26;
1625         cone[3] = 21;
1626         cone[4] = 18;
1627         cone[5] = 17;
1628         cone[6] = 24;
1629         cone[7] = 27;
1630         PetscCall(DMPlexSetCone(dm, 12, cone));
1631         cone[0] = 30;
1632         cone[1] = 28;
1633         cone[2] = 20;
1634         cone[3] = 19;
1635         cone[4] = 29;
1636         cone[5] = 15;
1637         cone[6] = 18;
1638         cone[7] = 27;
1639         PetscCall(DMPlexSetCone(dm, 13, cone));
1640         cone[0] = 30;
1641         cone[1] = 19;
1642         cone[2] = 22;
1643         cone[3] = 25;
1644         cone[4] = 29;
1645         cone[5] = 23;
1646         cone[6] = 16;
1647         cone[7] = 15;
1648         PetscCall(DMPlexSetCone(dm, 14, cone));
1649       } else {
1650         cone[0] = 5;
1651         cone[1] = 8;
1652         cone[2] = 7;
1653         cone[3] = 6;
1654         cone[4] = 9;
1655         cone[5] = 12;
1656         cone[6] = 11;
1657         cone[7] = 10;
1658         PetscCall(DMPlexSetCone(dm, 0, cone));
1659         cone[0] = 6;
1660         cone[1] = 7;
1661         cone[2] = 14;
1662         cone[3] = 13;
1663         cone[4] = 12;
1664         cone[5] = 15;
1665         cone[6] = 16;
1666         cone[7] = 11;
1667         PetscCall(DMPlexSetCone(dm, 1, cone));
1668         cone[0] = 8;
1669         cone[1] = 17;
1670         cone[2] = 14;
1671         cone[3] = 7;
1672         cone[4] = 10;
1673         cone[5] = 11;
1674         cone[6] = 16;
1675         cone[7] = 18;
1676         PetscCall(DMPlexSetCone(dm, 2, cone));
1677         cone[0] = 19;
1678         cone[1] = 17;
1679         cone[2] = 8;
1680         cone[3] = 5;
1681         cone[4] = 20;
1682         cone[5] = 9;
1683         cone[6] = 10;
1684         cone[7] = 18;
1685         PetscCall(DMPlexSetCone(dm, 3, cone));
1686         cone[0] = 19;
1687         cone[1] = 5;
1688         cone[2] = 6;
1689         cone[3] = 13;
1690         cone[4] = 20;
1691         cone[5] = 15;
1692         cone[6] = 12;
1693         cone[7] = 9;
1694         PetscCall(DMPlexSetCone(dm, 4, cone));
1695       }
1696     }
1697     PetscCall(DMPlexSymmetrize(dm));
1698     PetscCall(DMPlexStratify(dm));
1699   }
1700   /* Create cube geometry */
1701   {
1702     Vec             coordinates;
1703     PetscSection    coordSection;
1704     PetscScalar    *coords;
1705     PetscInt        coordSize, v;
1706     const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1707     const PetscReal ds2 = dis / 2.0;
1708 
1709     /* Build coordinates */
1710     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1711     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1712     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1713     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1714     for (v = numCells; v < numCells + numVertices; ++v) {
1715       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1716       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1717     }
1718     PetscCall(PetscSectionSetUp(coordSection));
1719     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1720     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1721     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1722     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1723     PetscCall(VecSetBlockSize(coordinates, dim));
1724     PetscCall(VecSetType(coordinates, VECSTANDARD));
1725     PetscCall(VecGetArray(coordinates, &coords));
1726     if (rank == 0) {
1727       coords[0 * dim + 0]  = -ds2;
1728       coords[0 * dim + 1]  = -ds2;
1729       coords[0 * dim + 2]  = 0.0;
1730       coords[1 * dim + 0]  = ds2;
1731       coords[1 * dim + 1]  = -ds2;
1732       coords[1 * dim + 2]  = 0.0;
1733       coords[2 * dim + 0]  = ds2;
1734       coords[2 * dim + 1]  = ds2;
1735       coords[2 * dim + 2]  = 0.0;
1736       coords[3 * dim + 0]  = -ds2;
1737       coords[3 * dim + 1]  = ds2;
1738       coords[3 * dim + 2]  = 0.0;
1739       coords[4 * dim + 0]  = -ds2;
1740       coords[4 * dim + 1]  = -ds2;
1741       coords[4 * dim + 2]  = 1.0;
1742       coords[5 * dim + 0]  = -ds2;
1743       coords[5 * dim + 1]  = ds2;
1744       coords[5 * dim + 2]  = 1.0;
1745       coords[6 * dim + 0]  = ds2;
1746       coords[6 * dim + 1]  = ds2;
1747       coords[6 * dim + 2]  = 1.0;
1748       coords[7 * dim + 0]  = ds2;
1749       coords[7 * dim + 1]  = -ds2;
1750       coords[7 * dim + 2]  = 1.0;
1751       coords[8 * dim + 0]  = dis;
1752       coords[8 * dim + 1]  = -dis;
1753       coords[8 * dim + 2]  = 0.0;
1754       coords[9 * dim + 0]  = dis;
1755       coords[9 * dim + 1]  = dis;
1756       coords[9 * dim + 2]  = 0.0;
1757       coords[10 * dim + 0] = dis;
1758       coords[10 * dim + 1] = -dis;
1759       coords[10 * dim + 2] = 1.0;
1760       coords[11 * dim + 0] = dis;
1761       coords[11 * dim + 1] = dis;
1762       coords[11 * dim + 2] = 1.0;
1763       coords[12 * dim + 0] = -dis;
1764       coords[12 * dim + 1] = dis;
1765       coords[12 * dim + 2] = 0.0;
1766       coords[13 * dim + 0] = -dis;
1767       coords[13 * dim + 1] = dis;
1768       coords[13 * dim + 2] = 1.0;
1769       coords[14 * dim + 0] = -dis;
1770       coords[14 * dim + 1] = -dis;
1771       coords[14 * dim + 2] = 0.0;
1772       coords[15 * dim + 0] = -dis;
1773       coords[15 * dim + 1] = -dis;
1774       coords[15 * dim + 2] = 1.0;
1775       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1776         /* 15 31 19 */ coords[16 * dim + 0] = -ds2;
1777         coords[16 * dim + 1]                = -ds2;
1778         coords[16 * dim + 2]                = 0.5;
1779         /* 16 32 22 */ coords[17 * dim + 0] = ds2;
1780         coords[17 * dim + 1]                = -ds2;
1781         coords[17 * dim + 2]                = 0.5;
1782         /* 17 33 21 */ coords[18 * dim + 0] = ds2;
1783         coords[18 * dim + 1]                = ds2;
1784         coords[18 * dim + 2]                = 0.5;
1785         /* 18 34 20 */ coords[19 * dim + 0] = -ds2;
1786         coords[19 * dim + 1]                = ds2;
1787         coords[19 * dim + 2]                = 0.5;
1788         /* 29 35 30 */ coords[20 * dim + 0] = -dis;
1789         coords[20 * dim + 1]                = -dis;
1790         coords[20 * dim + 2]                = 0.5;
1791         /* 23 36 25 */ coords[21 * dim + 0] = dis;
1792         coords[21 * dim + 1]                = -dis;
1793         coords[21 * dim + 2]                = 0.5;
1794         /* 24 37 26 */ coords[22 * dim + 0] = dis;
1795         coords[22 * dim + 1]                = dis;
1796         coords[22 * dim + 2]                = 0.5;
1797         /* 27 38 28 */ coords[23 * dim + 0] = -dis;
1798         coords[23 * dim + 1]                = dis;
1799         coords[23 * dim + 2]                = 0.5;
1800       }
1801     }
1802     PetscCall(VecRestoreArray(coordinates, &coords));
1803     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1804     PetscCall(VecDestroy(&coordinates));
1805   }
1806   /* Create periodicity */
1807   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1808     PetscReal L[3]       = {-1., -1., 0.};
1809     PetscReal maxCell[3] = {-1., -1., 0.};
1810     PetscReal lower[3]   = {0.0, 0.0, 0.0};
1811     PetscReal upper[3]   = {1.0, 1.0, 1.5};
1812     PetscInt  numZCells  = 3;
1813 
1814     L[2]       = upper[2] - lower[2];
1815     maxCell[2] = 1.1 * (L[2] / numZCells);
1816     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1817   }
1818   {
1819     DM          cdm;
1820     PetscDS     cds;
1821     PetscScalar c[2] = {1.0, 1.0};
1822 
1823     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToCylinder));
1824     PetscCall(DMGetCoordinateDM(dm, &cdm));
1825     PetscCall(DMGetDS(cdm, &cds));
1826     PetscCall(PetscDSSetConstants(cds, 2, c));
1827   }
1828   /* Wait for coordinate creation before doing in-place modification */
1829   PetscCall(DMPlexInterpolateInPlace_Internal(dm));
1830   PetscFunctionReturn(0);
1831 }
1832 
1833 /*@
1834   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
1835 
1836   Collective
1837 
1838   Input Parameters:
1839 + comm      - The communicator for the DM object
1840 - periodicZ - The boundary type for the Z direction
1841 
1842   Output Parameter:
1843 . dm  - The DM object
1844 
1845   Note:
1846   Here is the output numbering looking from the bottom of the cylinder:
1847 $       17-----14
1848 $        |     |
1849 $        |  2  |
1850 $        |     |
1851 $ 17-----8-----7-----14
1852 $  |     |     |     |
1853 $  |  3  |  0  |  1  |
1854 $  |     |     |     |
1855 $ 19-----5-----6-----13
1856 $        |     |
1857 $        |  4  |
1858 $        |     |
1859 $       19-----13
1860 $
1861 $ and up through the top
1862 $
1863 $       18-----16
1864 $        |     |
1865 $        |  2  |
1866 $        |     |
1867 $ 18----10----11-----16
1868 $  |     |     |     |
1869 $  |  3  |  0  |  1  |
1870 $  |     |     |     |
1871 $ 20-----9----12-----15
1872 $        |     |
1873 $        |  4  |
1874 $        |     |
1875 $       20-----15
1876 
1877   Level: beginner
1878 
1879 .seealso: `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1880 @*/
1881 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, DMBoundaryType periodicZ, DM *dm) {
1882   PetscFunctionBegin;
1883   PetscValidPointer(dm, 3);
1884   PetscCall(DMCreate(comm, dm));
1885   PetscCall(DMSetType(*dm, DMPLEX));
1886   PetscCall(DMPlexCreateHexCylinderMesh_Internal(*dm, periodicZ));
1887   PetscFunctionReturn(0);
1888 }
1889 
1890 static PetscErrorCode DMPlexCreateWedgeCylinderMesh_Internal(DM dm, PetscInt n, PetscBool interpolate) {
1891   const PetscInt dim = 3;
1892   PetscInt       numCells, numVertices, v;
1893   PetscMPIInt    rank;
1894 
1895   PetscFunctionBegin;
1896   PetscCheck(n >= 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %" PetscInt_FMT " cannot be negative", n);
1897   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1898   PetscCall(DMSetDimension(dm, dim));
1899   /* Must create the celltype label here so that we do not automatically try to compute the types */
1900   PetscCall(DMCreateLabel(dm, "celltype"));
1901   /* Create topology */
1902   {
1903     PetscInt cone[6], c;
1904 
1905     numCells    = rank == 0 ? n : 0;
1906     numVertices = rank == 0 ? 2 * (n + 1) : 0;
1907     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1908     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
1909     PetscCall(DMSetUp(dm));
1910     for (c = 0; c < numCells; c++) {
1911       cone[0] = c + n * 1;
1912       cone[1] = (c + 1) % n + n * 1;
1913       cone[2] = 0 + 3 * n;
1914       cone[3] = c + n * 2;
1915       cone[4] = (c + 1) % n + n * 2;
1916       cone[5] = 1 + 3 * n;
1917       PetscCall(DMPlexSetCone(dm, c, cone));
1918       PetscCall(DMPlexSetCellType(dm, c, DM_POLYTOPE_TRI_PRISM_TENSOR));
1919     }
1920     PetscCall(DMPlexSymmetrize(dm));
1921     PetscCall(DMPlexStratify(dm));
1922   }
1923   for (v = numCells; v < numCells + numVertices; ++v) { PetscCall(DMPlexSetCellType(dm, v, DM_POLYTOPE_POINT)); }
1924   /* Create cylinder geometry */
1925   {
1926     Vec          coordinates;
1927     PetscSection coordSection;
1928     PetscScalar *coords;
1929     PetscInt     coordSize, c;
1930 
1931     /* Build coordinates */
1932     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1933     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1934     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1935     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1936     for (v = numCells; v < numCells + numVertices; ++v) {
1937       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1938       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1939     }
1940     PetscCall(PetscSectionSetUp(coordSection));
1941     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1942     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1943     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1944     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1945     PetscCall(VecSetBlockSize(coordinates, dim));
1946     PetscCall(VecSetType(coordinates, VECSTANDARD));
1947     PetscCall(VecGetArray(coordinates, &coords));
1948     for (c = 0; c < numCells; c++) {
1949       coords[(c + 0 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
1950       coords[(c + 0 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
1951       coords[(c + 0 * n) * dim + 2] = 1.0;
1952       coords[(c + 1 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
1953       coords[(c + 1 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
1954       coords[(c + 1 * n) * dim + 2] = 0.0;
1955     }
1956     if (rank == 0) {
1957       coords[(2 * n + 0) * dim + 0] = 0.0;
1958       coords[(2 * n + 0) * dim + 1] = 0.0;
1959       coords[(2 * n + 0) * dim + 2] = 1.0;
1960       coords[(2 * n + 1) * dim + 0] = 0.0;
1961       coords[(2 * n + 1) * dim + 1] = 0.0;
1962       coords[(2 * n + 1) * dim + 2] = 0.0;
1963     }
1964     PetscCall(VecRestoreArray(coordinates, &coords));
1965     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1966     PetscCall(VecDestroy(&coordinates));
1967   }
1968   /* Interpolate */
1969   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
1970   PetscFunctionReturn(0);
1971 }
1972 
1973 /*@
1974   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
1975 
1976   Collective
1977 
1978   Input Parameters:
1979 + comm - The communicator for the DM object
1980 . n    - The number of wedges around the origin
1981 - interpolate - Create edges and faces
1982 
1983   Output Parameter:
1984 . dm  - The DM object
1985 
1986   Level: beginner
1987 
1988 .seealso: `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1989 @*/
1990 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm) {
1991   PetscFunctionBegin;
1992   PetscValidPointer(dm, 4);
1993   PetscCall(DMCreate(comm, dm));
1994   PetscCall(DMSetType(*dm, DMPLEX));
1995   PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(*dm, n, interpolate));
1996   PetscFunctionReturn(0);
1997 }
1998 
1999 static inline PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[]) {
2000   PetscReal prod = 0.0;
2001   PetscInt  i;
2002   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
2003   return PetscSqrtReal(prod);
2004 }
2005 static inline PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[]) {
2006   PetscReal prod = 0.0;
2007   PetscInt  i;
2008   for (i = 0; i < dim; ++i) prod += x[i] * y[i];
2009   return prod;
2010 }
2011 
2012 /* The first constant is the sphere radius */
2013 static void snapToSphere(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[]) {
2014   PetscReal r     = PetscRealPart(constants[0]);
2015   PetscReal norm2 = 0.0, fac;
2016   PetscInt  n     = uOff[1] - uOff[0], d;
2017 
2018   for (d = 0; d < n; ++d) norm2 += PetscSqr(PetscRealPart(u[d]));
2019   fac = r / PetscSqrtReal(norm2);
2020   for (d = 0; d < n; ++d) f0[d] = u[d] * fac;
2021 }
2022 
2023 static PetscErrorCode DMPlexCreateSphereMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, PetscReal R) {
2024   const PetscInt embedDim = dim + 1;
2025   PetscSection   coordSection;
2026   Vec            coordinates;
2027   PetscScalar   *coords;
2028   PetscReal     *coordsIn;
2029   PetscInt       numCells, numEdges, numVerts, firstVertex, v, firstEdge, coordSize, d, c, e;
2030   PetscMPIInt    rank;
2031 
2032   PetscFunctionBegin;
2033   PetscValidLogicalCollectiveBool(dm, simplex, 3);
2034   PetscCall(DMSetDimension(dm, dim));
2035   PetscCall(DMSetCoordinateDim(dm, dim + 1));
2036   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2037   switch (dim) {
2038   case 2:
2039     if (simplex) {
2040       const PetscReal radius    = PetscSqrtReal(1 + PETSC_PHI * PETSC_PHI) / (1.0 + PETSC_PHI);
2041       const PetscReal edgeLen   = 2.0 / (1.0 + PETSC_PHI) * (R / radius);
2042       const PetscInt  degree    = 5;
2043       PetscReal       vertex[3] = {0.0, 1.0 / (1.0 + PETSC_PHI), PETSC_PHI / (1.0 + PETSC_PHI)};
2044       PetscInt        s[3]      = {1, 1, 1};
2045       PetscInt        cone[3];
2046       PetscInt       *graph, p, i, j, k;
2047 
2048       vertex[0] *= R / radius;
2049       vertex[1] *= R / radius;
2050       vertex[2] *= R / radius;
2051       numCells    = rank == 0 ? 20 : 0;
2052       numVerts    = rank == 0 ? 12 : 0;
2053       firstVertex = numCells;
2054       /* Use icosahedron, which for a R-sphere has coordinates which are all cyclic permutations of
2055 
2056            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
2057 
2058          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2059          length is then given by 2/(1+\phi) = 2 * 0.38197 = 0.76393.
2060       */
2061       /* Construct vertices */
2062       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2063       if (rank == 0) {
2064         for (p = 0, i = 0; p < embedDim; ++p) {
2065           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2066             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2067               for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertex[(d + p) % embedDim];
2068               ++i;
2069             }
2070           }
2071         }
2072       }
2073       /* Construct graph */
2074       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2075       for (i = 0; i < numVerts; ++i) {
2076         for (j = 0, k = 0; j < numVerts; ++j) {
2077           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2078             graph[i * numVerts + j] = 1;
2079             ++k;
2080           }
2081         }
2082         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid icosahedron, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2083       }
2084       /* Build Topology */
2085       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2086       for (c = 0; c < numCells; c++) { PetscCall(DMPlexSetConeSize(dm, c, embedDim)); }
2087       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2088       /* Cells */
2089       for (i = 0, c = 0; i < numVerts; ++i) {
2090         for (j = 0; j < i; ++j) {
2091           for (k = 0; k < j; ++k) {
2092             if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i]) {
2093               cone[0] = firstVertex + i;
2094               cone[1] = firstVertex + j;
2095               cone[2] = firstVertex + k;
2096               /* Check orientation */
2097               {
2098                 const PetscInt epsilon[3][3][3] = {
2099                   {{0, 0, 0},  {0, 0, 1},  {0, -1, 0}},
2100                   {{0, 0, -1}, {0, 0, 0},  {1, 0, 0} },
2101                   {{0, 1, 0},  {-1, 0, 0}, {0, 0, 0} }
2102                 };
2103                 PetscReal normal[3];
2104                 PetscInt  e, f;
2105 
2106                 for (d = 0; d < embedDim; ++d) {
2107                   normal[d] = 0.0;
2108                   for (e = 0; e < embedDim; ++e) {
2109                     for (f = 0; f < embedDim; ++f) { normal[d] += epsilon[d][e][f] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]); }
2110                   }
2111                 }
2112                 if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2113                   PetscInt tmp = cone[1];
2114                   cone[1]      = cone[2];
2115                   cone[2]      = tmp;
2116                 }
2117               }
2118               PetscCall(DMPlexSetCone(dm, c++, cone));
2119             }
2120           }
2121         }
2122       }
2123       PetscCall(DMPlexSymmetrize(dm));
2124       PetscCall(DMPlexStratify(dm));
2125       PetscCall(PetscFree(graph));
2126     } else {
2127       /*
2128         12-21--13
2129          |     |
2130         25  4  24
2131          |     |
2132   12-25--9-16--8-24--13
2133    |     |     |     |
2134   23  5 17  0 15  3  22
2135    |     |     |     |
2136   10-20--6-14--7-19--11
2137          |     |
2138         20  1  19
2139          |     |
2140         10-18--11
2141          |     |
2142         23  2  22
2143          |     |
2144         12-21--13
2145        */
2146       PetscInt cone[4], ornt[4];
2147 
2148       numCells    = rank == 0 ? 6 : 0;
2149       numEdges    = rank == 0 ? 12 : 0;
2150       numVerts    = rank == 0 ? 8 : 0;
2151       firstVertex = numCells;
2152       firstEdge   = numCells + numVerts;
2153       /* Build Topology */
2154       PetscCall(DMPlexSetChart(dm, 0, numCells + numEdges + numVerts));
2155       for (c = 0; c < numCells; c++) { PetscCall(DMPlexSetConeSize(dm, c, 4)); }
2156       for (e = firstEdge; e < firstEdge + numEdges; ++e) { PetscCall(DMPlexSetConeSize(dm, e, 2)); }
2157       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2158       if (rank == 0) {
2159         /* Cell 0 */
2160         cone[0] = 14;
2161         cone[1] = 15;
2162         cone[2] = 16;
2163         cone[3] = 17;
2164         PetscCall(DMPlexSetCone(dm, 0, cone));
2165         ornt[0] = 0;
2166         ornt[1] = 0;
2167         ornt[2] = 0;
2168         ornt[3] = 0;
2169         PetscCall(DMPlexSetConeOrientation(dm, 0, ornt));
2170         /* Cell 1 */
2171         cone[0] = 18;
2172         cone[1] = 19;
2173         cone[2] = 14;
2174         cone[3] = 20;
2175         PetscCall(DMPlexSetCone(dm, 1, cone));
2176         ornt[0] = 0;
2177         ornt[1] = 0;
2178         ornt[2] = -1;
2179         ornt[3] = 0;
2180         PetscCall(DMPlexSetConeOrientation(dm, 1, ornt));
2181         /* Cell 2 */
2182         cone[0] = 21;
2183         cone[1] = 22;
2184         cone[2] = 18;
2185         cone[3] = 23;
2186         PetscCall(DMPlexSetCone(dm, 2, cone));
2187         ornt[0] = 0;
2188         ornt[1] = 0;
2189         ornt[2] = -1;
2190         ornt[3] = 0;
2191         PetscCall(DMPlexSetConeOrientation(dm, 2, ornt));
2192         /* Cell 3 */
2193         cone[0] = 19;
2194         cone[1] = 22;
2195         cone[2] = 24;
2196         cone[3] = 15;
2197         PetscCall(DMPlexSetCone(dm, 3, cone));
2198         ornt[0] = -1;
2199         ornt[1] = -1;
2200         ornt[2] = 0;
2201         ornt[3] = -1;
2202         PetscCall(DMPlexSetConeOrientation(dm, 3, ornt));
2203         /* Cell 4 */
2204         cone[0] = 16;
2205         cone[1] = 24;
2206         cone[2] = 21;
2207         cone[3] = 25;
2208         PetscCall(DMPlexSetCone(dm, 4, cone));
2209         ornt[0] = -1;
2210         ornt[1] = -1;
2211         ornt[2] = -1;
2212         ornt[3] = 0;
2213         PetscCall(DMPlexSetConeOrientation(dm, 4, ornt));
2214         /* Cell 5 */
2215         cone[0] = 20;
2216         cone[1] = 17;
2217         cone[2] = 25;
2218         cone[3] = 23;
2219         PetscCall(DMPlexSetCone(dm, 5, cone));
2220         ornt[0] = -1;
2221         ornt[1] = -1;
2222         ornt[2] = -1;
2223         ornt[3] = -1;
2224         PetscCall(DMPlexSetConeOrientation(dm, 5, ornt));
2225         /* Edges */
2226         cone[0] = 6;
2227         cone[1] = 7;
2228         PetscCall(DMPlexSetCone(dm, 14, cone));
2229         cone[0] = 7;
2230         cone[1] = 8;
2231         PetscCall(DMPlexSetCone(dm, 15, cone));
2232         cone[0] = 8;
2233         cone[1] = 9;
2234         PetscCall(DMPlexSetCone(dm, 16, cone));
2235         cone[0] = 9;
2236         cone[1] = 6;
2237         PetscCall(DMPlexSetCone(dm, 17, cone));
2238         cone[0] = 10;
2239         cone[1] = 11;
2240         PetscCall(DMPlexSetCone(dm, 18, cone));
2241         cone[0] = 11;
2242         cone[1] = 7;
2243         PetscCall(DMPlexSetCone(dm, 19, cone));
2244         cone[0] = 6;
2245         cone[1] = 10;
2246         PetscCall(DMPlexSetCone(dm, 20, cone));
2247         cone[0] = 12;
2248         cone[1] = 13;
2249         PetscCall(DMPlexSetCone(dm, 21, cone));
2250         cone[0] = 13;
2251         cone[1] = 11;
2252         PetscCall(DMPlexSetCone(dm, 22, cone));
2253         cone[0] = 10;
2254         cone[1] = 12;
2255         PetscCall(DMPlexSetCone(dm, 23, cone));
2256         cone[0] = 13;
2257         cone[1] = 8;
2258         PetscCall(DMPlexSetCone(dm, 24, cone));
2259         cone[0] = 12;
2260         cone[1] = 9;
2261         PetscCall(DMPlexSetCone(dm, 25, cone));
2262       }
2263       PetscCall(DMPlexSymmetrize(dm));
2264       PetscCall(DMPlexStratify(dm));
2265       /* Build coordinates */
2266       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2267       if (rank == 0) {
2268         coordsIn[0 * embedDim + 0] = -R;
2269         coordsIn[0 * embedDim + 1] = R;
2270         coordsIn[0 * embedDim + 2] = -R;
2271         coordsIn[1 * embedDim + 0] = R;
2272         coordsIn[1 * embedDim + 1] = R;
2273         coordsIn[1 * embedDim + 2] = -R;
2274         coordsIn[2 * embedDim + 0] = R;
2275         coordsIn[2 * embedDim + 1] = -R;
2276         coordsIn[2 * embedDim + 2] = -R;
2277         coordsIn[3 * embedDim + 0] = -R;
2278         coordsIn[3 * embedDim + 1] = -R;
2279         coordsIn[3 * embedDim + 2] = -R;
2280         coordsIn[4 * embedDim + 0] = -R;
2281         coordsIn[4 * embedDim + 1] = R;
2282         coordsIn[4 * embedDim + 2] = R;
2283         coordsIn[5 * embedDim + 0] = R;
2284         coordsIn[5 * embedDim + 1] = R;
2285         coordsIn[5 * embedDim + 2] = R;
2286         coordsIn[6 * embedDim + 0] = -R;
2287         coordsIn[6 * embedDim + 1] = -R;
2288         coordsIn[6 * embedDim + 2] = R;
2289         coordsIn[7 * embedDim + 0] = R;
2290         coordsIn[7 * embedDim + 1] = -R;
2291         coordsIn[7 * embedDim + 2] = R;
2292       }
2293     }
2294     break;
2295   case 3:
2296     if (simplex) {
2297       const PetscReal edgeLen         = 1.0 / PETSC_PHI;
2298       PetscReal       vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
2299       PetscReal       vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
2300       PetscReal       vertexC[4]      = {0.5, 0.5 * PETSC_PHI, 0.5 / PETSC_PHI, 0.0};
2301       const PetscInt  degree          = 12;
2302       PetscInt        s[4]            = {1, 1, 1};
2303       PetscInt        evenPerm[12][4] = {
2304                {0, 1, 2, 3},
2305                {0, 2, 3, 1},
2306                {0, 3, 1, 2},
2307                {1, 0, 3, 2},
2308                {1, 2, 0, 3},
2309                {1, 3, 2, 0},
2310                {2, 0, 1, 3},
2311                {2, 1, 3, 0},
2312                {2, 3, 0, 1},
2313                {3, 0, 2, 1},
2314                {3, 1, 0, 2},
2315                {3, 2, 1, 0}
2316       };
2317       PetscInt  cone[4];
2318       PetscInt *graph, p, i, j, k, l;
2319 
2320       vertexA[0] *= R;
2321       vertexA[1] *= R;
2322       vertexA[2] *= R;
2323       vertexA[3] *= R;
2324       vertexB[0] *= R;
2325       vertexB[1] *= R;
2326       vertexB[2] *= R;
2327       vertexB[3] *= R;
2328       vertexC[0] *= R;
2329       vertexC[1] *= R;
2330       vertexC[2] *= R;
2331       vertexC[3] *= R;
2332       numCells    = rank == 0 ? 600 : 0;
2333       numVerts    = rank == 0 ? 120 : 0;
2334       firstVertex = numCells;
2335       /* Use the 600-cell, which for a unit sphere has coordinates which are
2336 
2337            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
2338                (\pm 1,    0,       0,      0)  all cyclic permutations   8
2339            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
2340 
2341          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2342          length is then given by 1/\phi = 0.61803.
2343 
2344          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
2345          http://mathworld.wolfram.com/600-Cell.html
2346       */
2347       /* Construct vertices */
2348       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2349       i = 0;
2350       if (rank == 0) {
2351         for (s[0] = -1; s[0] < 2; s[0] += 2) {
2352           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2353             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2354               for (s[3] = -1; s[3] < 2; s[3] += 2) {
2355                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[d] * vertexA[d];
2356                 ++i;
2357               }
2358             }
2359           }
2360         }
2361         for (p = 0; p < embedDim; ++p) {
2362           s[1] = s[2] = s[3] = 1;
2363           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2364             for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertexB[(d + p) % embedDim];
2365             ++i;
2366           }
2367         }
2368         for (p = 0; p < 12; ++p) {
2369           s[3] = 1;
2370           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2371             for (s[1] = -1; s[1] < 2; s[1] += 2) {
2372               for (s[2] = -1; s[2] < 2; s[2] += 2) {
2373                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[evenPerm[p][d]] * vertexC[evenPerm[p][d]];
2374                 ++i;
2375               }
2376             }
2377           }
2378         }
2379       }
2380       PetscCheck(i == numVerts, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertices %" PetscInt_FMT " != %" PetscInt_FMT, i, numVerts);
2381       /* Construct graph */
2382       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2383       for (i = 0; i < numVerts; ++i) {
2384         for (j = 0, k = 0; j < numVerts; ++j) {
2385           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2386             graph[i * numVerts + j] = 1;
2387             ++k;
2388           }
2389         }
2390         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2391       }
2392       /* Build Topology */
2393       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2394       for (c = 0; c < numCells; c++) { PetscCall(DMPlexSetConeSize(dm, c, embedDim)); }
2395       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2396       /* Cells */
2397       if (rank == 0) {
2398         for (i = 0, c = 0; i < numVerts; ++i) {
2399           for (j = 0; j < i; ++j) {
2400             for (k = 0; k < j; ++k) {
2401               for (l = 0; l < k; ++l) {
2402                 if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i] && graph[l * numVerts + i] && graph[l * numVerts + j] && graph[l * numVerts + k]) {
2403                   cone[0] = firstVertex + i;
2404                   cone[1] = firstVertex + j;
2405                   cone[2] = firstVertex + k;
2406                   cone[3] = firstVertex + l;
2407                   /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
2408                   {
2409                     const PetscInt epsilon[4][4][4][4] = {
2410                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, -1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 0, 0}, {0, 1, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 1, 0}, {0, -1, 0, 0}, {0, 0, 0, 0}}},
2411 
2412                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}}, {{0, 0, -1, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}}},
2413 
2414                       {{{0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, 0, 0}, {0, -1, 0, 0}}, {{0, 0, 0, -1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 1, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}},
2415 
2416                       {{{0, 0, 0, 0}, {0, 0, -1, 0}, {0, 1, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 1, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}}, {{0, -1, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}} }
2417                     };
2418                     PetscReal normal[4];
2419                     PetscInt  e, f, g;
2420 
2421                     for (d = 0; d < embedDim; ++d) {
2422                       normal[d] = 0.0;
2423                       for (e = 0; e < embedDim; ++e) {
2424                         for (f = 0; f < embedDim; ++f) {
2425                           for (g = 0; g < embedDim; ++g) {
2426                             normal[d] += epsilon[d][e][f][g] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]) * (coordsIn[l * embedDim + f] - coordsIn[i * embedDim + f]);
2427                           }
2428                         }
2429                       }
2430                     }
2431                     if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2432                       PetscInt tmp = cone[1];
2433                       cone[1]      = cone[2];
2434                       cone[2]      = tmp;
2435                     }
2436                   }
2437                   PetscCall(DMPlexSetCone(dm, c++, cone));
2438                 }
2439               }
2440             }
2441           }
2442         }
2443       }
2444       PetscCall(DMPlexSymmetrize(dm));
2445       PetscCall(DMPlexStratify(dm));
2446       PetscCall(PetscFree(graph));
2447       break;
2448     }
2449   default: SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unsupported dimension for sphere: %" PetscInt_FMT, dim);
2450   }
2451   /* Create coordinates */
2452   PetscCall(DMGetCoordinateSection(dm, &coordSection));
2453   PetscCall(PetscSectionSetNumFields(coordSection, 1));
2454   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, embedDim));
2455   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVerts));
2456   for (v = firstVertex; v < firstVertex + numVerts; ++v) {
2457     PetscCall(PetscSectionSetDof(coordSection, v, embedDim));
2458     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, embedDim));
2459   }
2460   PetscCall(PetscSectionSetUp(coordSection));
2461   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2462   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2463   PetscCall(VecSetBlockSize(coordinates, embedDim));
2464   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2465   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2466   PetscCall(VecSetType(coordinates, VECSTANDARD));
2467   PetscCall(VecGetArray(coordinates, &coords));
2468   for (v = 0; v < numVerts; ++v)
2469     for (d = 0; d < embedDim; ++d) { coords[v * embedDim + d] = coordsIn[v * embedDim + d]; }
2470   PetscCall(VecRestoreArray(coordinates, &coords));
2471   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2472   PetscCall(VecDestroy(&coordinates));
2473   PetscCall(PetscFree(coordsIn));
2474   {
2475     DM          cdm;
2476     PetscDS     cds;
2477     PetscScalar c = R;
2478 
2479     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToSphere));
2480     PetscCall(DMGetCoordinateDM(dm, &cdm));
2481     PetscCall(DMGetDS(cdm, &cds));
2482     PetscCall(PetscDSSetConstants(cds, 1, &c));
2483   }
2484   /* Wait for coordinate creation before doing in-place modification */
2485   if (simplex) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2486   PetscFunctionReturn(0);
2487 }
2488 
2489 typedef void (*TPSEvaluateFunc)(const PetscReal[], PetscReal *, PetscReal[], PetscReal (*)[3]);
2490 
2491 /*
2492  The Schwarz P implicit surface is
2493 
2494      f(x) = cos(x0) + cos(x1) + cos(x2) = 0
2495 */
2496 static void TPSEvaluate_SchwarzP(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3]) {
2497   PetscReal c[3] = {PetscCosReal(y[0] * PETSC_PI), PetscCosReal(y[1] * PETSC_PI), PetscCosReal(y[2] * PETSC_PI)};
2498   PetscReal g[3] = {-PetscSinReal(y[0] * PETSC_PI), -PetscSinReal(y[1] * PETSC_PI), -PetscSinReal(y[2] * PETSC_PI)};
2499   f[0]           = c[0] + c[1] + c[2];
2500   for (PetscInt i = 0; i < 3; i++) {
2501     grad[i] = PETSC_PI * g[i];
2502     for (PetscInt j = 0; j < 3; j++) { hess[i][j] = (i == j) ? -PetscSqr(PETSC_PI) * c[i] : 0.; }
2503   }
2504 }
2505 
2506 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2507 static PetscErrorCode TPSExtrudeNormalFunc_SchwarzP(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx) {
2508   for (PetscInt i = 0; i < 3; i++) { u[i] = -PETSC_PI * PetscSinReal(x[i] * PETSC_PI); }
2509   return 0;
2510 }
2511 
2512 /*
2513  The Gyroid implicit surface is
2514 
2515  f(x,y,z) = sin(pi * x) * cos (pi * (y + 1/2))  + sin(pi * (y + 1/2)) * cos(pi * (z + 1/4)) + sin(pi * (z + 1/4)) * cos(pi * x)
2516 
2517 */
2518 static void TPSEvaluate_Gyroid(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3]) {
2519   PetscReal s[3] = {PetscSinReal(PETSC_PI * y[0]), PetscSinReal(PETSC_PI * (y[1] + .5)), PetscSinReal(PETSC_PI * (y[2] + .25))};
2520   PetscReal c[3] = {PetscCosReal(PETSC_PI * y[0]), PetscCosReal(PETSC_PI * (y[1] + .5)), PetscCosReal(PETSC_PI * (y[2] + .25))};
2521   f[0]           = s[0] * c[1] + s[1] * c[2] + s[2] * c[0];
2522   grad[0]        = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2523   grad[1]        = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2524   grad[2]        = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2525   hess[0][0]     = -PetscSqr(PETSC_PI) * (s[0] * c[1] + s[2] * c[0]);
2526   hess[0][1]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2527   hess[0][2]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2528   hess[1][0]     = -PetscSqr(PETSC_PI) * (s[1] * c[2] + s[0] * c[1]);
2529   hess[1][1]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2530   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2531   hess[2][0]     = -PetscSqr(PETSC_PI) * (s[2] * c[0] + s[1] * c[2]);
2532   hess[2][1]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2533   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2534 }
2535 
2536 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2537 static PetscErrorCode TPSExtrudeNormalFunc_Gyroid(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx) {
2538   PetscReal s[3] = {PetscSinReal(PETSC_PI * x[0]), PetscSinReal(PETSC_PI * (x[1] + .5)), PetscSinReal(PETSC_PI * (x[2] + .25))};
2539   PetscReal c[3] = {PetscCosReal(PETSC_PI * x[0]), PetscCosReal(PETSC_PI * (x[1] + .5)), PetscCosReal(PETSC_PI * (x[2] + .25))};
2540   u[0]           = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2541   u[1]           = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2542   u[2]           = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2543   return 0;
2544 }
2545 
2546 /*
2547    We wish to solve
2548 
2549          min_y || y - x ||^2  subject to f(y) = 0
2550 
2551    Let g(y) = grad(f).  The minimization problem is equivalent to asking to satisfy
2552    f(y) = 0 and (y-x) is parallel to g(y).  We do this by using Householder QR to obtain a basis for the
2553    tangent space and ask for both components in the tangent space to be zero.
2554 
2555    Take g to be a column vector and compute the "full QR" factorization Q R = g,
2556    where Q = I - 2 n n^T is a symmetric orthogonal matrix.
2557    The first column of Q is parallel to g so the remaining two columns span the null space.
2558    Let Qn = Q[:,1:] be those remaining columns.  Then Qn Qn^T is an orthogonal projector into the tangent space.
2559    Since Q is symmetric, this is equivalent to multipyling by Q and taking the last two entries.
2560    In total, we have a system of 3 equations in 3 unknowns:
2561 
2562      f(y) = 0                       1 equation
2563      Qn^T (y - x) = 0               2 equations
2564 
2565    Here, we compute the residual and Jacobian of this system.
2566 */
2567 static void TPSNearestPointResJac(TPSEvaluateFunc feval, const PetscScalar x[], const PetscScalar y[], PetscScalar res[], PetscScalar J[]) {
2568   PetscReal yreal[3] = {PetscRealPart(y[0]), PetscRealPart(y[1]), PetscRealPart(y[2])};
2569   PetscReal d[3]     = {PetscRealPart(y[0] - x[0]), PetscRealPart(y[1] - x[1]), PetscRealPart(y[2] - x[2])};
2570   PetscReal f, grad[3], n[3], norm, norm_y[3], nd, nd_y[3], sign;
2571   PetscReal n_y[3][3] = {
2572     {0, 0, 0},
2573     {0, 0, 0},
2574     {0, 0, 0}
2575   };
2576 
2577   feval(yreal, &f, grad, n_y);
2578 
2579   for (PetscInt i = 0; i < 3; i++) n[i] = grad[i];
2580   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2581   for (PetscInt i = 0; i < 3; i++) { norm_y[i] = 1. / norm * n[i] * n_y[i][i]; }
2582 
2583   // Define the Householder reflector
2584   sign = n[0] >= 0 ? 1. : -1.;
2585   n[0] += norm * sign;
2586   for (PetscInt i = 0; i < 3; i++) n_y[0][i] += norm_y[i] * sign;
2587 
2588   norm      = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2589   norm_y[0] = 1. / norm * (n[0] * n_y[0][0]);
2590   norm_y[1] = 1. / norm * (n[0] * n_y[0][1] + n[1] * n_y[1][1]);
2591   norm_y[2] = 1. / norm * (n[0] * n_y[0][2] + n[2] * n_y[2][2]);
2592 
2593   for (PetscInt i = 0; i < 3; i++) {
2594     n[i] /= norm;
2595     for (PetscInt j = 0; j < 3; j++) {
2596       // note that n[i] is n_old[i]/norm when executing the code below
2597       n_y[i][j] = n_y[i][j] / norm - n[i] / norm * norm_y[j];
2598     }
2599   }
2600 
2601   nd = n[0] * d[0] + n[1] * d[1] + n[2] * d[2];
2602   for (PetscInt i = 0; i < 3; i++) nd_y[i] = n[i] + n_y[0][i] * d[0] + n_y[1][i] * d[1] + n_y[2][i] * d[2];
2603 
2604   res[0] = f;
2605   res[1] = d[1] - 2 * n[1] * nd;
2606   res[2] = d[2] - 2 * n[2] * nd;
2607   // J[j][i] is J_{ij} (column major)
2608   for (PetscInt j = 0; j < 3; j++) {
2609     J[0 + j * 3] = grad[j];
2610     J[1 + j * 3] = (j == 1) * 1. - 2 * (n_y[1][j] * nd + n[1] * nd_y[j]);
2611     J[2 + j * 3] = (j == 2) * 1. - 2 * (n_y[2][j] * nd + n[2] * nd_y[j]);
2612   }
2613 }
2614 
2615 /*
2616    Project x to the nearest point on the implicit surface using Newton's method.
2617 */
2618 static PetscErrorCode TPSNearestPoint(TPSEvaluateFunc feval, PetscScalar x[]) {
2619   PetscScalar y[3] = {x[0], x[1], x[2]}; // Initial guess
2620 
2621   PetscFunctionBegin;
2622   for (PetscInt iter = 0; iter < 10; iter++) {
2623     PetscScalar res[3], J[9];
2624     PetscReal   resnorm;
2625     TPSNearestPointResJac(feval, x, y, res, J);
2626     resnorm = PetscSqrtReal(PetscSqr(PetscRealPart(res[0])) + PetscSqr(PetscRealPart(res[1])) + PetscSqr(PetscRealPart(res[2])));
2627     if (0) { // Turn on this monitor if you need to confirm quadratic convergence
2628       PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%" PetscInt_FMT "] res [%g %g %g]\n", iter, (double)PetscRealPart(res[0]), (double)PetscRealPart(res[1]), (double)PetscRealPart(res[2])));
2629     }
2630     if (resnorm < PETSC_SMALL) break;
2631 
2632     // Take the Newton step
2633     PetscCall(PetscKernel_A_gets_inverse_A_3(J, 0., PETSC_FALSE, NULL));
2634     PetscKernel_v_gets_v_minus_A_times_w_3(y, J, res);
2635   }
2636   for (PetscInt i = 0; i < 3; i++) x[i] = y[i];
2637   PetscFunctionReturn(0);
2638 }
2639 
2640 const char *const DMPlexTPSTypes[] = {"SCHWARZ_P", "GYROID", "DMPlexTPSType", "DMPLEX_TPS_", NULL};
2641 
2642 static PetscErrorCode DMPlexCreateTPSMesh_Internal(DM dm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness) {
2643   PetscMPIInt rank;
2644   PetscInt    topoDim = 2, spaceDim = 3, numFaces = 0, numVertices = 0, numEdges = 0;
2645   PetscInt(*edges)[2] = NULL, *edgeSets = NULL;
2646   PetscInt            *cells_flat = NULL;
2647   PetscReal           *vtxCoords  = NULL;
2648   TPSEvaluateFunc      evalFunc   = NULL;
2649   PetscSimplePointFunc normalFunc = NULL;
2650   DMLabel              label;
2651 
2652   PetscFunctionBegin;
2653   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2654   PetscCheck((layers != 0) ^ (thickness == 0.), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Layers %" PetscInt_FMT " must be nonzero iff thickness %g is nonzero", layers, (double)thickness);
2655   switch (tpstype) {
2656   case DMPLEX_TPS_SCHWARZ_P:
2657     PetscCheck(!periodic || (periodic[0] == DM_BOUNDARY_NONE && periodic[1] == DM_BOUNDARY_NONE && periodic[2] == DM_BOUNDARY_NONE), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Schwarz P does not support periodic meshes");
2658     if (rank == 0) {
2659       PetscInt(*cells)[6][4][4] = NULL; // [junction, junction-face, cell, conn]
2660       PetscInt  Njunctions = 0, Ncuts = 0, Npipes[3], vcount;
2661       PetscReal L = 1;
2662 
2663       Npipes[0]   = (extent[0] + 1) * extent[1] * extent[2];
2664       Npipes[1]   = extent[0] * (extent[1] + 1) * extent[2];
2665       Npipes[2]   = extent[0] * extent[1] * (extent[2] + 1);
2666       Njunctions  = extent[0] * extent[1] * extent[2];
2667       Ncuts       = 2 * (extent[0] * extent[1] + extent[1] * extent[2] + extent[2] * extent[0]);
2668       numVertices = 4 * (Npipes[0] + Npipes[1] + Npipes[2]) + 8 * Njunctions;
2669       PetscCall(PetscMalloc1(3 * numVertices, &vtxCoords));
2670       PetscCall(PetscMalloc1(Njunctions, &cells));
2671       PetscCall(PetscMalloc1(Ncuts * 4, &edges));
2672       PetscCall(PetscMalloc1(Ncuts * 4, &edgeSets));
2673       // x-normal pipes
2674       vcount = 0;
2675       for (PetscInt i = 0; i < extent[0] + 1; i++) {
2676         for (PetscInt j = 0; j < extent[1]; j++) {
2677           for (PetscInt k = 0; k < extent[2]; k++) {
2678             for (PetscInt l = 0; l < 4; l++) {
2679               vtxCoords[vcount++] = (2 * i - 1) * L;
2680               vtxCoords[vcount++] = 2 * j * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2681               vtxCoords[vcount++] = 2 * k * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2682             }
2683           }
2684         }
2685       }
2686       // y-normal pipes
2687       for (PetscInt i = 0; i < extent[0]; i++) {
2688         for (PetscInt j = 0; j < extent[1] + 1; j++) {
2689           for (PetscInt k = 0; k < extent[2]; k++) {
2690             for (PetscInt l = 0; l < 4; l++) {
2691               vtxCoords[vcount++] = 2 * i * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2692               vtxCoords[vcount++] = (2 * j - 1) * L;
2693               vtxCoords[vcount++] = 2 * k * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2694             }
2695           }
2696         }
2697       }
2698       // z-normal pipes
2699       for (PetscInt i = 0; i < extent[0]; i++) {
2700         for (PetscInt j = 0; j < extent[1]; j++) {
2701           for (PetscInt k = 0; k < extent[2] + 1; k++) {
2702             for (PetscInt l = 0; l < 4; l++) {
2703               vtxCoords[vcount++] = 2 * i * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2704               vtxCoords[vcount++] = 2 * j * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2705               vtxCoords[vcount++] = (2 * k - 1) * L;
2706             }
2707           }
2708         }
2709       }
2710       // junctions
2711       for (PetscInt i = 0; i < extent[0]; i++) {
2712         for (PetscInt j = 0; j < extent[1]; j++) {
2713           for (PetscInt k = 0; k < extent[2]; k++) {
2714             const PetscInt J = (i * extent[1] + j) * extent[2] + k, Jvoff = (Npipes[0] + Npipes[1] + Npipes[2]) * 4 + J * 8;
2715             PetscCheck(vcount / 3 == Jvoff, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected vertex count");
2716             for (PetscInt ii = 0; ii < 2; ii++) {
2717               for (PetscInt jj = 0; jj < 2; jj++) {
2718                 for (PetscInt kk = 0; kk < 2; kk++) {
2719                   double Ls           = (1 - sqrt(2) / 4) * L;
2720                   vtxCoords[vcount++] = 2 * i * L + (2 * ii - 1) * Ls;
2721                   vtxCoords[vcount++] = 2 * j * L + (2 * jj - 1) * Ls;
2722                   vtxCoords[vcount++] = 2 * k * L + (2 * kk - 1) * Ls;
2723                 }
2724               }
2725             }
2726             const PetscInt jfaces[3][2][4] = {
2727               {{3, 1, 0, 2}, {7, 5, 4, 6}}, // x-aligned
2728               {{5, 4, 0, 1}, {7, 6, 2, 3}}, // y-aligned
2729               {{6, 2, 0, 4}, {7, 3, 1, 5}}  // z-aligned
2730             };
2731             const PetscInt pipe_lo[3] = {// vertex numbers of pipes
2732                                          ((i * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + Npipes[0] + Npipes[1]) * 4};
2733             const PetscInt pipe_hi[3] = {// vertex numbers of pipes
2734                                          (((i + 1) * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j + 1) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + 1 + Npipes[0] + Npipes[1]) * 4};
2735             for (PetscInt dir = 0; dir < 3; dir++) { // x,y,z
2736               const PetscInt ijk[3] = {i, j, k};
2737               for (PetscInt l = 0; l < 4; l++) { // rotations
2738                 cells[J][dir * 2 + 0][l][0] = pipe_lo[dir] + l;
2739                 cells[J][dir * 2 + 0][l][1] = Jvoff + jfaces[dir][0][l];
2740                 cells[J][dir * 2 + 0][l][2] = Jvoff + jfaces[dir][0][(l - 1 + 4) % 4];
2741                 cells[J][dir * 2 + 0][l][3] = pipe_lo[dir] + (l - 1 + 4) % 4;
2742                 cells[J][dir * 2 + 1][l][0] = Jvoff + jfaces[dir][1][l];
2743                 cells[J][dir * 2 + 1][l][1] = pipe_hi[dir] + l;
2744                 cells[J][dir * 2 + 1][l][2] = pipe_hi[dir] + (l - 1 + 4) % 4;
2745                 cells[J][dir * 2 + 1][l][3] = Jvoff + jfaces[dir][1][(l - 1 + 4) % 4];
2746                 if (ijk[dir] == 0) {
2747                   edges[numEdges][0] = pipe_lo[dir] + l;
2748                   edges[numEdges][1] = pipe_lo[dir] + (l + 1) % 4;
2749                   edgeSets[numEdges] = dir * 2 + 1;
2750                   numEdges++;
2751                 }
2752                 if (ijk[dir] + 1 == extent[dir]) {
2753                   edges[numEdges][0] = pipe_hi[dir] + l;
2754                   edges[numEdges][1] = pipe_hi[dir] + (l + 1) % 4;
2755                   edgeSets[numEdges] = dir * 2 + 2;
2756                   numEdges++;
2757                 }
2758               }
2759             }
2760           }
2761         }
2762       }
2763       PetscCheck(numEdges == Ncuts * 4, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Edge count %" PetscInt_FMT " incompatible with number of cuts %" PetscInt_FMT, numEdges, Ncuts);
2764       numFaces   = 24 * Njunctions;
2765       cells_flat = cells[0][0][0];
2766     }
2767     evalFunc   = TPSEvaluate_SchwarzP;
2768     normalFunc = TPSExtrudeNormalFunc_SchwarzP;
2769     break;
2770   case DMPLEX_TPS_GYROID:
2771     if (rank == 0) {
2772       // This is a coarse mesh approximation of the gyroid shifted to being the zero of the level set
2773       //
2774       //     sin(pi*x)*cos(pi*(y+1/2)) + sin(pi*(y+1/2))*cos(pi*(z+1/4)) + sin(pi*(z+1/4))*cos(x)
2775       //
2776       // on the cell [0,2]^3.
2777       //
2778       // Think about dividing that cell into four columns, and focus on the column [0,1]x[0,1]x[0,2].
2779       // If you looked at the gyroid in that column at different slices of z you would see that it kind of spins
2780       // like a boomerang:
2781       //
2782       //     z = 0          z = 1/4        z = 1/2        z = 3/4     //
2783       //     -----          -------        -------        -------     //
2784       //                                                              //
2785       //     +       +      +       +      +       +      +   \   +   //
2786       //      \                                   /            \      //
2787       //       \            `-_   _-'            /              }     //
2788       //        *-_            `-'            _-'              /      //
2789       //     +     `-+      +       +      +-'     +      +   /   +   //
2790       //                                                              //
2791       //                                                              //
2792       //     z = 1          z = 5/4        z = 3/2        z = 7/4     //
2793       //     -----          -------        -------        -------     //
2794       //                                                              //
2795       //     +-_     +      +       +      +     _-+      +   /   +   //
2796       //        `-_            _-_            _-`            /        //
2797       //           \        _-'   `-_        /              {         //
2798       //            \                       /                \        //
2799       //     +       +      +       +      +       +      +   \   +   //
2800       //
2801       //
2802       // This course mesh approximates each of these slices by two line segments,
2803       // and then connects the segments in consecutive layers with quadrilateral faces.
2804       // All of the end points of the segments are multiples of 1/4 except for the
2805       // point * in the picture for z = 0 above and the similar points in other layers.
2806       // That point is at (gamma, gamma, 0), where gamma is calculated below.
2807       //
2808       // The column  [1,2]x[1,2]x[0,2] looks the same as this column;
2809       // The columns [1,2]x[0,1]x[0,2] and [0,1]x[1,2]x[0,2] are mirror images.
2810       //
2811       // As for how this method turned into the names given to the vertices:
2812       // that was not systematic, it was just the way it worked out in my handwritten notes.
2813 
2814       PetscInt       facesPerBlock = 64;
2815       PetscInt       vertsPerBlock = 56;
2816       PetscInt       extentPlus[3];
2817       PetscInt       numBlocks, numBlocksPlus;
2818       const PetscInt A = 0, B = 1, C = 2, D = 3, E = 4, F = 5, G = 6, H = 7, II = 8, J = 9, K = 10, L = 11, M = 12, N = 13, O = 14, P = 15, Q = 16, R = 17, S = 18, T = 19, U = 20, V = 21, W = 22, X = 23, Y = 24, Z = 25, Ap = 26, Bp = 27, Cp = 28, Dp = 29, Ep = 30, Fp = 31, Gp = 32, Hp = 33, Ip = 34, Jp = 35, Kp = 36, Lp = 37, Mp = 38, Np = 39, Op = 40, Pp = 41, Qp = 42, Rp = 43, Sp = 44, Tp = 45, Up = 46, Vp = 47, Wp = 48, Xp = 49, Yp = 50, Zp = 51, Aq = 52, Bq = 53, Cq = 54, Dq = 55;
2819       const PetscInt pattern[64][4] = {
2820   /* face to vertex within the coarse discretization of a single gyroid block */
2821   /* layer 0 */
2822         {A,           C,           K,           G          },
2823         {C,           B,           II,          K          },
2824         {D,           A,           H,           L          },
2825         {B + 56 * 1,  D,           L,           J          },
2826         {E,           B + 56 * 1,  J,           N          },
2827         {A + 56 * 2,  E,           N,           H + 56 * 2 },
2828         {F,           A + 56 * 2,  G + 56 * 2,  M          },
2829         {B,           F,           M,           II         },
2830  /* layer 1 */
2831         {G,           K,           Q,           O          },
2832         {K,           II,          P,           Q          },
2833         {L,           H,           O + 56 * 1,  R          },
2834         {J,           L,           R,           P          },
2835         {N,           J,           P,           S          },
2836         {H + 56 * 2,  N,           S,           O + 56 * 3 },
2837         {M,           G + 56 * 2,  O + 56 * 2,  T          },
2838         {II,          M,           T,           P          },
2839  /* layer 2 */
2840         {O,           Q,           Y,           U          },
2841         {Q,           P,           W,           Y          },
2842         {R,           O + 56 * 1,  U + 56 * 1,  Ap         },
2843         {P,           R,           Ap,          W          },
2844         {S,           P,           X,           Bp         },
2845         {O + 56 * 3,  S,           Bp,          V + 56 * 1 },
2846         {T,           O + 56 * 2,  V,           Z          },
2847         {P,           T,           Z,           X          },
2848  /* layer 3 */
2849         {U,           Y,           Ep,          Dp         },
2850         {Y,           W,           Cp,          Ep         },
2851         {Ap,          U + 56 * 1,  Dp + 56 * 1, Gp         },
2852         {W,           Ap,          Gp,          Cp         },
2853         {Bp,          X,           Cp + 56 * 2, Fp         },
2854         {V + 56 * 1,  Bp,          Fp,          Dp + 56 * 1},
2855         {Z,           V,           Dp,          Hp         },
2856         {X,           Z,           Hp,          Cp + 56 * 2},
2857  /* layer 4 */
2858         {Dp,          Ep,          Mp,          Kp         },
2859         {Ep,          Cp,          Ip,          Mp         },
2860         {Gp,          Dp + 56 * 1, Lp,          Np         },
2861         {Cp,          Gp,          Np,          Jp         },
2862         {Fp,          Cp + 56 * 2, Jp + 56 * 2, Pp         },
2863         {Dp + 56 * 1, Fp,          Pp,          Lp         },
2864         {Hp,          Dp,          Kp,          Op         },
2865         {Cp + 56 * 2, Hp,          Op,          Ip + 56 * 2},
2866  /* layer 5 */
2867         {Kp,          Mp,          Sp,          Rp         },
2868         {Mp,          Ip,          Qp,          Sp         },
2869         {Np,          Lp,          Rp,          Tp         },
2870         {Jp,          Np,          Tp,          Qp + 56 * 1},
2871         {Pp,          Jp + 56 * 2, Qp + 56 * 3, Up         },
2872         {Lp,          Pp,          Up,          Rp         },
2873         {Op,          Kp,          Rp,          Vp         },
2874         {Ip + 56 * 2, Op,          Vp,          Qp + 56 * 2},
2875  /* layer 6 */
2876         {Rp,          Sp,          Aq,          Yp         },
2877         {Sp,          Qp,          Wp,          Aq         },
2878         {Tp,          Rp,          Yp,          Cq         },
2879         {Qp + 56 * 1, Tp,          Cq,          Wp + 56 * 1},
2880         {Up,          Qp + 56 * 3, Xp + 56 * 1, Dq         },
2881         {Rp,          Up,          Dq,          Zp         },
2882         {Vp,          Rp,          Zp,          Bq         },
2883         {Qp + 56 * 2, Vp,          Bq,          Xp         },
2884  /* layer 7 (the top is the periodic image of the bottom of layer 0) */
2885         {Yp,          Aq,          C + 56 * 4,  A + 56 * 4 },
2886         {Aq,          Wp,          B + 56 * 4,  C + 56 * 4 },
2887         {Cq,          Yp,          A + 56 * 4,  D + 56 * 4 },
2888         {Wp + 56 * 1, Cq,          D + 56 * 4,  B + 56 * 5 },
2889         {Dq,          Xp + 56 * 1, B + 56 * 5,  E + 56 * 4 },
2890         {Zp,          Dq,          E + 56 * 4,  A + 56 * 6 },
2891         {Bq,          Zp,          A + 56 * 6,  F + 56 * 4 },
2892         {Xp,          Bq,          F + 56 * 4,  B + 56 * 4 }
2893       };
2894       const PetscReal gamma                = PetscAcosReal((PetscSqrtReal(3.) - 1.) / PetscSqrtReal(2.)) / PETSC_PI;
2895       const PetscReal patternCoords[56][3] = {
2896         {1.,        0.,        0.  }, /* A  */
2897         {0.,        1.,        0.  }, /* B  */
2898         {gamma,     gamma,     0.  }, /* C  */
2899         {1 + gamma, 1 - gamma, 0.  }, /* D  */
2900         {2 - gamma, 2 - gamma, 0.  }, /* E  */
2901         {1 - gamma, 1 + gamma, 0.  }, /* F  */
2902 
2903         {.5,        0,         .25 }, /* G  */
2904         {1.5,       0.,        .25 }, /* H  */
2905         {.5,        1.,        .25 }, /* II */
2906         {1.5,       1.,        .25 }, /* J  */
2907         {.25,       .5,        .25 }, /* K  */
2908         {1.25,      .5,        .25 }, /* L  */
2909         {.75,       1.5,       .25 }, /* M  */
2910         {1.75,      1.5,       .25 }, /* N  */
2911 
2912         {0.,        0.,        .5  }, /* O  */
2913         {1.,        1.,        .5  }, /* P  */
2914         {gamma,     1 - gamma, .5  }, /* Q  */
2915         {1 + gamma, gamma,     .5  }, /* R  */
2916         {2 - gamma, 1 + gamma, .5  }, /* S  */
2917         {1 - gamma, 2 - gamma, .5  }, /* T  */
2918 
2919         {0.,        .5,        .75 }, /* U  */
2920         {0.,        1.5,       .75 }, /* V  */
2921         {1.,        .5,        .75 }, /* W  */
2922         {1.,        1.5,       .75 }, /* X  */
2923         {.5,        .75,       .75 }, /* Y  */
2924         {.5,        1.75,      .75 }, /* Z  */
2925         {1.5,       .25,       .75 }, /* Ap */
2926         {1.5,       1.25,      .75 }, /* Bp */
2927 
2928         {1.,        0.,        1.  }, /* Cp */
2929         {0.,        1.,        1.  }, /* Dp */
2930         {1 - gamma, 1 - gamma, 1.  }, /* Ep */
2931         {1 + gamma, 1 + gamma, 1.  }, /* Fp */
2932         {2 - gamma, gamma,     1.  }, /* Gp */
2933         {gamma,     2 - gamma, 1.  }, /* Hp */
2934 
2935         {.5,        0.,        1.25}, /* Ip */
2936         {1.5,       0.,        1.25}, /* Jp */
2937         {.5,        1.,        1.25}, /* Kp */
2938         {1.5,       1.,        1.25}, /* Lp */
2939         {.75,       .5,        1.25}, /* Mp */
2940         {1.75,      .5,        1.25}, /* Np */
2941         {.25,       1.5,       1.25}, /* Op */
2942         {1.25,      1.5,       1.25}, /* Pp */
2943 
2944         {0.,        0.,        1.5 }, /* Qp */
2945         {1.,        1.,        1.5 }, /* Rp */
2946         {1 - gamma, gamma,     1.5 }, /* Sp */
2947         {2 - gamma, 1 - gamma, 1.5 }, /* Tp */
2948         {1 + gamma, 2 - gamma, 1.5 }, /* Up */
2949         {gamma,     1 + gamma, 1.5 }, /* Vp */
2950 
2951         {0.,        .5,        1.75}, /* Wp */
2952         {0.,        1.5,       1.75}, /* Xp */
2953         {1.,        .5,        1.75}, /* Yp */
2954         {1.,        1.5,       1.75}, /* Zp */
2955         {.5,        .25,       1.75}, /* Aq */
2956         {.5,        1.25,      1.75}, /* Bq */
2957         {1.5,       .75,       1.75}, /* Cq */
2958         {1.5,       1.75,      1.75}, /* Dq */
2959       };
2960       PetscInt(*cells)[64][4] = NULL;
2961       PetscBool *seen;
2962       PetscInt  *vertToTrueVert;
2963       PetscInt   count;
2964 
2965       for (PetscInt i = 0; i < 3; i++) extentPlus[i] = extent[i] + 1;
2966       numBlocks = 1;
2967       for (PetscInt i = 0; i < 3; i++) numBlocks *= extent[i];
2968       numBlocksPlus = 1;
2969       for (PetscInt i = 0; i < 3; i++) numBlocksPlus *= extentPlus[i];
2970       numFaces = numBlocks * facesPerBlock;
2971       PetscCall(PetscMalloc1(numBlocks, &cells));
2972       PetscCall(PetscCalloc1(numBlocksPlus * vertsPerBlock, &seen));
2973       for (PetscInt k = 0; k < extent[2]; k++) {
2974         for (PetscInt j = 0; j < extent[1]; j++) {
2975           for (PetscInt i = 0; i < extent[0]; i++) {
2976             for (PetscInt f = 0; f < facesPerBlock; f++) {
2977               for (PetscInt v = 0; v < 4; v++) {
2978                 PetscInt vertRaw     = pattern[f][v];
2979                 PetscInt blockidx    = vertRaw / 56;
2980                 PetscInt patternvert = vertRaw % 56;
2981                 PetscInt xplus       = (blockidx & 1);
2982                 PetscInt yplus       = (blockidx & 2) >> 1;
2983                 PetscInt zplus       = (blockidx & 4) >> 2;
2984                 PetscInt zcoord      = (periodic && periodic[2] == DM_BOUNDARY_PERIODIC) ? ((k + zplus) % extent[2]) : (k + zplus);
2985                 PetscInt ycoord      = (periodic && periodic[1] == DM_BOUNDARY_PERIODIC) ? ((j + yplus) % extent[1]) : (j + yplus);
2986                 PetscInt xcoord      = (periodic && periodic[0] == DM_BOUNDARY_PERIODIC) ? ((i + xplus) % extent[0]) : (i + xplus);
2987                 PetscInt vert        = ((zcoord * extentPlus[1] + ycoord) * extentPlus[0] + xcoord) * 56 + patternvert;
2988 
2989                 cells[(k * extent[1] + j) * extent[0] + i][f][v] = vert;
2990                 seen[vert]                                       = PETSC_TRUE;
2991               }
2992             }
2993           }
2994         }
2995       }
2996       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++)
2997         if (seen[i]) numVertices++;
2998       count = 0;
2999       PetscCall(PetscMalloc1(numBlocksPlus * vertsPerBlock, &vertToTrueVert));
3000       PetscCall(PetscMalloc1(numVertices * 3, &vtxCoords));
3001       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) vertToTrueVert[i] = -1;
3002       for (PetscInt k = 0; k < extentPlus[2]; k++) {
3003         for (PetscInt j = 0; j < extentPlus[1]; j++) {
3004           for (PetscInt i = 0; i < extentPlus[0]; i++) {
3005             for (PetscInt v = 0; v < vertsPerBlock; v++) {
3006               PetscInt vIdx = ((k * extentPlus[1] + j) * extentPlus[0] + i) * vertsPerBlock + v;
3007 
3008               if (seen[vIdx]) {
3009                 PetscInt thisVert;
3010 
3011                 vertToTrueVert[vIdx] = thisVert = count++;
3012 
3013                 for (PetscInt d = 0; d < 3; d++) vtxCoords[3 * thisVert + d] = patternCoords[v][d];
3014                 vtxCoords[3 * thisVert + 0] += i * 2;
3015                 vtxCoords[3 * thisVert + 1] += j * 2;
3016                 vtxCoords[3 * thisVert + 2] += k * 2;
3017               }
3018             }
3019           }
3020         }
3021       }
3022       for (PetscInt i = 0; i < numBlocks; i++) {
3023         for (PetscInt f = 0; f < facesPerBlock; f++) {
3024           for (PetscInt v = 0; v < 4; v++) { cells[i][f][v] = vertToTrueVert[cells[i][f][v]]; }
3025         }
3026       }
3027       PetscCall(PetscFree(vertToTrueVert));
3028       PetscCall(PetscFree(seen));
3029       cells_flat = cells[0][0];
3030       numEdges   = 0;
3031       for (PetscInt i = 0; i < numFaces; i++) {
3032         for (PetscInt e = 0; e < 4; e++) {
3033           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3034           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3035 
3036           for (PetscInt d = 0; d < 3; d++) {
3037             if (!periodic || periodic[0] != DM_BOUNDARY_PERIODIC) {
3038               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) numEdges++;
3039               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) numEdges++;
3040             }
3041           }
3042         }
3043       }
3044       PetscCall(PetscMalloc1(numEdges, &edges));
3045       PetscCall(PetscMalloc1(numEdges, &edgeSets));
3046       for (PetscInt edge = 0, i = 0; i < numFaces; i++) {
3047         for (PetscInt e = 0; e < 4; e++) {
3048           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3049           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3050 
3051           for (PetscInt d = 0; d < 3; d++) {
3052             if (!periodic || periodic[d] != DM_BOUNDARY_PERIODIC) {
3053               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) {
3054                 edges[edge][0]   = ev[0];
3055                 edges[edge][1]   = ev[1];
3056                 edgeSets[edge++] = 2 * d;
3057               }
3058               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) {
3059                 edges[edge][0]   = ev[0];
3060                 edges[edge][1]   = ev[1];
3061                 edgeSets[edge++] = 2 * d + 1;
3062               }
3063             }
3064           }
3065         }
3066       }
3067     }
3068     evalFunc   = TPSEvaluate_Gyroid;
3069     normalFunc = TPSExtrudeNormalFunc_Gyroid;
3070     break;
3071   }
3072 
3073   PetscCall(DMSetDimension(dm, topoDim));
3074   if (rank == 0) PetscCall(DMPlexBuildFromCellList(dm, numFaces, numVertices, 4, cells_flat));
3075   else PetscCall(DMPlexBuildFromCellList(dm, 0, 0, 0, NULL));
3076   PetscCall(PetscFree(cells_flat));
3077   {
3078     DM idm;
3079     PetscCall(DMPlexInterpolate(dm, &idm));
3080     PetscCall(DMPlexReplace_Static(dm, &idm));
3081   }
3082   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, vtxCoords));
3083   else PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, NULL));
3084   PetscCall(PetscFree(vtxCoords));
3085 
3086   PetscCall(DMCreateLabel(dm, "Face Sets"));
3087   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3088   for (PetscInt e = 0; e < numEdges; e++) {
3089     PetscInt        njoin;
3090     const PetscInt *join, verts[] = {numFaces + edges[e][0], numFaces + edges[e][1]};
3091     PetscCall(DMPlexGetJoin(dm, 2, verts, &njoin, &join));
3092     PetscCheck(njoin == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected unique join of vertices %" PetscInt_FMT " and %" PetscInt_FMT, edges[e][0], edges[e][1]);
3093     PetscCall(DMLabelSetValue(label, join[0], edgeSets[e]));
3094     PetscCall(DMPlexRestoreJoin(dm, 2, verts, &njoin, &join));
3095   }
3096   PetscCall(PetscFree(edges));
3097   PetscCall(PetscFree(edgeSets));
3098   if (tps_distribute) {
3099     DM               pdm = NULL;
3100     PetscPartitioner part;
3101 
3102     PetscCall(DMPlexGetPartitioner(dm, &part));
3103     PetscCall(PetscPartitionerSetFromOptions(part));
3104     PetscCall(DMPlexDistribute(dm, 0, NULL, &pdm));
3105     if (pdm) { PetscCall(DMPlexReplace_Static(dm, &pdm)); }
3106     // Do not auto-distribute again
3107     PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
3108   }
3109 
3110   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3111   for (PetscInt refine = 0; refine < refinements; refine++) {
3112     PetscInt     m;
3113     DM           dmf;
3114     Vec          X;
3115     PetscScalar *x;
3116     PetscCall(DMRefine(dm, MPI_COMM_NULL, &dmf));
3117     PetscCall(DMPlexReplace_Static(dm, &dmf));
3118 
3119     PetscCall(DMGetCoordinatesLocal(dm, &X));
3120     PetscCall(VecGetLocalSize(X, &m));
3121     PetscCall(VecGetArray(X, &x));
3122     for (PetscInt i = 0; i < m; i += 3) { PetscCall(TPSNearestPoint(evalFunc, &x[i])); }
3123     PetscCall(VecRestoreArray(X, &x));
3124   }
3125 
3126   // Face Sets has already been propagated to new vertices during refinement; this propagates to the initial vertices.
3127   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3128   PetscCall(DMPlexLabelComplete(dm, label));
3129 
3130   if (thickness > 0) {
3131     DM              edm, cdm, ecdm;
3132     DMPlexTransform tr;
3133     const char     *prefix;
3134     PetscOptions    options;
3135     // Code from DMPlexExtrude
3136     PetscCall(DMPlexTransformCreate(PetscObjectComm((PetscObject)dm), &tr));
3137     PetscCall(DMPlexTransformSetDM(tr, dm));
3138     PetscCall(DMPlexTransformSetType(tr, DMPLEXEXTRUDE));
3139     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
3140     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)tr, prefix));
3141     PetscCall(PetscObjectGetOptions((PetscObject)dm, &options));
3142     PetscCall(PetscObjectSetOptions((PetscObject)tr, options));
3143     PetscCall(DMPlexTransformExtrudeSetLayers(tr, layers));
3144     PetscCall(DMPlexTransformExtrudeSetThickness(tr, thickness));
3145     PetscCall(DMPlexTransformExtrudeSetTensor(tr, PETSC_FALSE));
3146     PetscCall(DMPlexTransformExtrudeSetSymmetric(tr, PETSC_TRUE));
3147     PetscCall(DMPlexTransformExtrudeSetNormalFunction(tr, normalFunc));
3148     PetscCall(DMPlexTransformSetFromOptions(tr));
3149     PetscCall(PetscObjectSetOptions((PetscObject)tr, NULL));
3150     PetscCall(DMPlexTransformSetUp(tr));
3151     PetscCall(PetscObjectViewFromOptions((PetscObject)tr, NULL, "-dm_plex_tps_transform_view"));
3152     PetscCall(DMPlexTransformApply(tr, dm, &edm));
3153     PetscCall(DMCopyDisc(dm, edm));
3154     PetscCall(DMGetCoordinateDM(dm, &cdm));
3155     PetscCall(DMGetCoordinateDM(edm, &ecdm));
3156     PetscCall(DMCopyDisc(cdm, ecdm));
3157     PetscCall(DMPlexTransformCreateDiscLabels(tr, edm));
3158     PetscCall(DMPlexTransformDestroy(&tr));
3159     if (edm) {
3160       ((DM_Plex *)edm->data)->printFEM    = ((DM_Plex *)dm->data)->printFEM;
3161       ((DM_Plex *)edm->data)->printL2     = ((DM_Plex *)dm->data)->printL2;
3162       ((DM_Plex *)edm->data)->printLocate = ((DM_Plex *)dm->data)->printLocate;
3163     }
3164     PetscCall(DMPlexReplace_Static(dm, &edm));
3165   }
3166   PetscFunctionReturn(0);
3167 }
3168 
3169 /*@
3170   DMPlexCreateTPSMesh - Create a distributed, interpolated mesh of a triply-periodic surface
3171 
3172   Collective
3173 
3174   Input Parameters:
3175 + comm   - The communicator for the DM object
3176 . tpstype - Type of triply-periodic surface
3177 . extent - Array of length 3 containing number of periods in each direction
3178 . periodic - array of length 3 with periodicity, or NULL for non-periodic
3179 . tps_distribute - Distribute 2D manifold mesh prior to refinement and extrusion (more scalable)
3180 . refinements - Number of factor-of-2 refinements of 2D manifold mesh
3181 . layers - Number of cell layers extruded in normal direction
3182 - thickness - Thickness in normal direction
3183 
3184   Output Parameter:
3185 . dm  - The DM object
3186 
3187   Notes:
3188   This meshes the surface of the Schwarz P or Gyroid surfaces.  Schwarz P is is the simplest member of the triply-periodic minimal surfaces.
3189   https://en.wikipedia.org/wiki/Schwarz_minimal_surface#Schwarz_P_(%22Primitive%22) and can be cut with "clean" boundaries.
3190   The Gyroid (https://en.wikipedia.org/wiki/Gyroid) is another triply-periodic minimal surface with applications in additive manufacturing; it is much more difficult to "cut" since there are no planes of symmetry.
3191   Our implementation creates a very coarse mesh of the surface and refines (by 4-way splitting) as many times as requested.
3192   On each refinement, all vertices are projected to their nearest point on the surface.
3193   This projection could readily be extended to related surfaces.
3194 
3195   The face (edge) sets for the Schwarz P surface are numbered 1(-x), 2(+x), 3(-y), 4(+y), 5(-z), 6(+z).
3196   When the mesh is refined, "Face Sets" contain the new vertices (created during refinement).  Use DMPlexLabelComplete() to propagate to coarse-level vertices.
3197 
3198   References:
3199 . * - Maskery et al, Insights into the mechanical properties of several triply periodic minimal surface lattice structures made by polymer additive manufacturing, 2017. https://doi.org/10.1016/j.polymer.2017.11.049
3200 
3201   Developer Notes:
3202   The Gyroid mesh does not currently mark boundary sets.
3203 
3204   Level: beginner
3205 
3206 .seealso: `DMPlexCreateSphereMesh()`, `DMSetType()`, `DMCreate()`
3207 @*/
3208 PetscErrorCode DMPlexCreateTPSMesh(MPI_Comm comm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness, DM *dm) {
3209   PetscFunctionBegin;
3210   PetscCall(DMCreate(comm, dm));
3211   PetscCall(DMSetType(*dm, DMPLEX));
3212   PetscCall(DMPlexCreateTPSMesh_Internal(*dm, tpstype, extent, periodic, tps_distribute, refinements, layers, thickness));
3213   PetscFunctionReturn(0);
3214 }
3215 
3216 /*@
3217   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
3218 
3219   Collective
3220 
3221   Input Parameters:
3222 + comm    - The communicator for the DM object
3223 . dim     - The dimension
3224 . simplex - Use simplices, or tensor product cells
3225 - R       - The radius
3226 
3227   Output Parameter:
3228 . dm  - The DM object
3229 
3230   Level: beginner
3231 
3232 .seealso: `DMPlexCreateBallMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3233 @*/
3234 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscReal R, DM *dm) {
3235   PetscFunctionBegin;
3236   PetscValidPointer(dm, 5);
3237   PetscCall(DMCreate(comm, dm));
3238   PetscCall(DMSetType(*dm, DMPLEX));
3239   PetscCall(DMPlexCreateSphereMesh_Internal(*dm, dim, simplex, R));
3240   PetscFunctionReturn(0);
3241 }
3242 
3243 static PetscErrorCode DMPlexCreateBallMesh_Internal(DM dm, PetscInt dim, PetscReal R) {
3244   DM      sdm, vol;
3245   DMLabel bdlabel;
3246 
3247   PetscFunctionBegin;
3248   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
3249   PetscCall(DMSetType(sdm, DMPLEX));
3250   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sdm, "bd_"));
3251   PetscCall(DMPlexCreateSphereMesh_Internal(sdm, dim - 1, PETSC_TRUE, R));
3252   PetscCall(DMSetFromOptions(sdm));
3253   PetscCall(DMViewFromOptions(sdm, NULL, "-dm_view"));
3254   PetscCall(DMPlexGenerate(sdm, NULL, PETSC_TRUE, &vol));
3255   PetscCall(DMDestroy(&sdm));
3256   PetscCall(DMPlexReplace_Static(dm, &vol));
3257   PetscCall(DMCreateLabel(dm, "marker"));
3258   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
3259   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
3260   PetscCall(DMPlexLabelComplete(dm, bdlabel));
3261   PetscFunctionReturn(0);
3262 }
3263 
3264 /*@
3265   DMPlexCreateBallMesh - Creates a simplex mesh on the d-dimensional ball, B^d.
3266 
3267   Collective
3268 
3269   Input Parameters:
3270 + comm  - The communicator for the DM object
3271 . dim   - The dimension
3272 - R     - The radius
3273 
3274   Output Parameter:
3275 . dm  - The DM object
3276 
3277   Options Database Keys:
3278 - bd_dm_refine - This will refine the surface mesh preserving the sphere geometry
3279 
3280   Level: beginner
3281 
3282 .seealso: `DMPlexCreateSphereMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3283 @*/
3284 PetscErrorCode DMPlexCreateBallMesh(MPI_Comm comm, PetscInt dim, PetscReal R, DM *dm) {
3285   PetscFunctionBegin;
3286   PetscCall(DMCreate(comm, dm));
3287   PetscCall(DMSetType(*dm, DMPLEX));
3288   PetscCall(DMPlexCreateBallMesh_Internal(*dm, dim, R));
3289   PetscFunctionReturn(0);
3290 }
3291 
3292 static PetscErrorCode DMPlexCreateReferenceCell_Internal(DM rdm, DMPolytopeType ct) {
3293   PetscFunctionBegin;
3294   switch (ct) {
3295   case DM_POLYTOPE_POINT: {
3296     PetscInt    numPoints[1]        = {1};
3297     PetscInt    coneSize[1]         = {0};
3298     PetscInt    cones[1]            = {0};
3299     PetscInt    coneOrientations[1] = {0};
3300     PetscScalar vertexCoords[1]     = {0.0};
3301 
3302     PetscCall(DMSetDimension(rdm, 0));
3303     PetscCall(DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3304   } break;
3305   case DM_POLYTOPE_SEGMENT: {
3306     PetscInt    numPoints[2]        = {2, 1};
3307     PetscInt    coneSize[3]         = {2, 0, 0};
3308     PetscInt    cones[2]            = {1, 2};
3309     PetscInt    coneOrientations[2] = {0, 0};
3310     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3311 
3312     PetscCall(DMSetDimension(rdm, 1));
3313     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3314   } break;
3315   case DM_POLYTOPE_POINT_PRISM_TENSOR: {
3316     PetscInt    numPoints[2]        = {2, 1};
3317     PetscInt    coneSize[3]         = {2, 0, 0};
3318     PetscInt    cones[2]            = {1, 2};
3319     PetscInt    coneOrientations[2] = {0, 0};
3320     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3321 
3322     PetscCall(DMSetDimension(rdm, 1));
3323     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3324   } break;
3325   case DM_POLYTOPE_TRIANGLE: {
3326     PetscInt    numPoints[2]        = {3, 1};
3327     PetscInt    coneSize[4]         = {3, 0, 0, 0};
3328     PetscInt    cones[3]            = {1, 2, 3};
3329     PetscInt    coneOrientations[3] = {0, 0, 0};
3330     PetscScalar vertexCoords[6]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0};
3331 
3332     PetscCall(DMSetDimension(rdm, 2));
3333     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3334   } break;
3335   case DM_POLYTOPE_QUADRILATERAL: {
3336     PetscInt    numPoints[2]        = {4, 1};
3337     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3338     PetscInt    cones[4]            = {1, 2, 3, 4};
3339     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3340     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0};
3341 
3342     PetscCall(DMSetDimension(rdm, 2));
3343     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3344   } break;
3345   case DM_POLYTOPE_SEG_PRISM_TENSOR: {
3346     PetscInt    numPoints[2]        = {4, 1};
3347     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3348     PetscInt    cones[4]            = {1, 2, 3, 4};
3349     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3350     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0};
3351 
3352     PetscCall(DMSetDimension(rdm, 2));
3353     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3354   } break;
3355   case DM_POLYTOPE_TETRAHEDRON: {
3356     PetscInt    numPoints[2]        = {4, 1};
3357     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3358     PetscInt    cones[4]            = {1, 2, 3, 4};
3359     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3360     PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0};
3361 
3362     PetscCall(DMSetDimension(rdm, 3));
3363     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3364   } break;
3365   case DM_POLYTOPE_HEXAHEDRON: {
3366     PetscInt    numPoints[2]        = {8, 1};
3367     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3368     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3369     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3370     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3371 
3372     PetscCall(DMSetDimension(rdm, 3));
3373     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3374   } break;
3375   case DM_POLYTOPE_TRI_PRISM: {
3376     PetscInt    numPoints[2]        = {6, 1};
3377     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3378     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3379     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3380     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3381 
3382     PetscCall(DMSetDimension(rdm, 3));
3383     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3384   } break;
3385   case DM_POLYTOPE_TRI_PRISM_TENSOR: {
3386     PetscInt    numPoints[2]        = {6, 1};
3387     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3388     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3389     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3390     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3391 
3392     PetscCall(DMSetDimension(rdm, 3));
3393     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3394   } break;
3395   case DM_POLYTOPE_QUAD_PRISM_TENSOR: {
3396     PetscInt    numPoints[2]        = {8, 1};
3397     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3398     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3399     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3400     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3401 
3402     PetscCall(DMSetDimension(rdm, 3));
3403     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3404   } break;
3405   case DM_POLYTOPE_PYRAMID: {
3406     PetscInt    numPoints[2]        = {5, 1};
3407     PetscInt    coneSize[6]         = {5, 0, 0, 0, 0, 0};
3408     PetscInt    cones[5]            = {1, 2, 3, 4, 5};
3409     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3410     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, 0.0, 0.0, 1.0};
3411 
3412     PetscCall(DMSetDimension(rdm, 3));
3413     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3414   } break;
3415   default: SETERRQ(PetscObjectComm((PetscObject)rdm), PETSC_ERR_ARG_WRONG, "Cannot create reference cell for cell type %s", DMPolytopeTypes[ct]);
3416   }
3417   {
3418     PetscInt Nv, v;
3419 
3420     /* Must create the celltype label here so that we do not automatically try to compute the types */
3421     PetscCall(DMCreateLabel(rdm, "celltype"));
3422     PetscCall(DMPlexSetCellType(rdm, 0, ct));
3423     PetscCall(DMPlexGetChart(rdm, NULL, &Nv));
3424     for (v = 1; v < Nv; ++v) PetscCall(DMPlexSetCellType(rdm, v, DM_POLYTOPE_POINT));
3425   }
3426   PetscCall(DMPlexInterpolateInPlace_Internal(rdm));
3427   PetscCall(PetscObjectSetName((PetscObject)rdm, DMPolytopeTypes[ct]));
3428   PetscFunctionReturn(0);
3429 }
3430 
3431 /*@
3432   DMPlexCreateReferenceCell - Create a DMPLEX with the appropriate FEM reference cell
3433 
3434   Collective
3435 
3436   Input Parameters:
3437 + comm - The communicator
3438 - ct   - The cell type of the reference cell
3439 
3440   Output Parameter:
3441 . refdm - The reference cell
3442 
3443   Level: intermediate
3444 
3445 .seealso: `DMPlexCreateReferenceCell()`, `DMPlexCreateBoxMesh()`
3446 @*/
3447 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, DMPolytopeType ct, DM *refdm) {
3448   PetscFunctionBegin;
3449   PetscCall(DMCreate(comm, refdm));
3450   PetscCall(DMSetType(*refdm, DMPLEX));
3451   PetscCall(DMPlexCreateReferenceCell_Internal(*refdm, ct));
3452   PetscFunctionReturn(0);
3453 }
3454 
3455 static PetscErrorCode DMPlexCreateBoundaryLabel_Private(DM dm, const char name[]) {
3456   DM        plex;
3457   DMLabel   label;
3458   PetscBool hasLabel;
3459 
3460   PetscFunctionBegin;
3461   PetscCall(DMHasLabel(dm, name, &hasLabel));
3462   if (hasLabel) PetscFunctionReturn(0);
3463   PetscCall(DMCreateLabel(dm, name));
3464   PetscCall(DMGetLabel(dm, name, &label));
3465   PetscCall(DMConvert(dm, DMPLEX, &plex));
3466   PetscCall(DMPlexMarkBoundaryFaces(plex, 1, label));
3467   PetscCall(DMPlexLabelComplete(plex, label));
3468   PetscCall(DMDestroy(&plex));
3469   PetscFunctionReturn(0);
3470 }
3471 
3472 const char *const DMPlexShapes[] = {"box", "box_surface", "ball", "sphere", "cylinder", "schwarz_p", "gyroid", "doublet", "unknown", "DMPlexShape", "DM_SHAPE_", NULL};
3473 
3474 static PetscErrorCode DMPlexCreateFromOptions_Internal(PetscOptionItems *PetscOptionsObject, PetscBool *useCoordSpace, DM dm) {
3475   DMPlexShape    shape   = DM_SHAPE_BOX;
3476   DMPolytopeType cell    = DM_POLYTOPE_TRIANGLE;
3477   PetscInt       dim     = 2;
3478   PetscBool      simplex = PETSC_TRUE, interpolate = PETSC_TRUE, adjCone = PETSC_FALSE, adjClosure = PETSC_TRUE, refDomain = PETSC_FALSE;
3479   PetscBool      flg, flg2, fflg, bdfflg, nameflg;
3480   MPI_Comm       comm;
3481   char           filename[PETSC_MAX_PATH_LEN]   = "<unspecified>";
3482   char           bdFilename[PETSC_MAX_PATH_LEN] = "<unspecified>";
3483   char           plexname[PETSC_MAX_PATH_LEN]   = "";
3484 
3485   PetscFunctionBegin;
3486   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3487   /* TODO Turn this into a registration interface */
3488   PetscCall(PetscOptionsString("-dm_plex_filename", "File containing a mesh", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &fflg));
3489   PetscCall(PetscOptionsString("-dm_plex_boundary_filename", "File containing a mesh boundary", "DMPlexCreateFromFile", bdFilename, bdFilename, sizeof(bdFilename), &bdfflg));
3490   PetscCall(PetscOptionsString("-dm_plex_name", "Name of the mesh in the file", "DMPlexCreateFromFile", plexname, plexname, sizeof(plexname), &nameflg));
3491   PetscCall(PetscOptionsEnum("-dm_plex_cell", "Cell shape", "", DMPolytopeTypes, (PetscEnum)cell, (PetscEnum *)&cell, NULL));
3492   PetscCall(PetscOptionsBool("-dm_plex_reference_cell_domain", "Use a reference cell domain", "", refDomain, &refDomain, NULL));
3493   PetscCall(PetscOptionsEnum("-dm_plex_shape", "Shape for built-in mesh", "", DMPlexShapes, (PetscEnum)shape, (PetscEnum *)&shape, &flg));
3494   PetscCall(PetscOptionsBoundedInt("-dm_plex_dim", "Topological dimension of the mesh", "DMGetDimension", dim, &dim, &flg, 0));
3495   PetscCheck(!(dim < 0) && !(dim > 3), comm, PETSC_ERR_ARG_OUTOFRANGE, "Dimension %" PetscInt_FMT " should be in [1, 3]", dim);
3496   PetscCall(PetscOptionsBool("-dm_plex_simplex", "Mesh cell shape", "", simplex, &simplex, &flg));
3497   PetscCall(PetscOptionsBool("-dm_plex_interpolate", "Flag to create edges and faces automatically", "", interpolate, &interpolate, &flg));
3498   PetscCall(PetscOptionsBool("-dm_plex_adj_cone", "Set adjacency direction", "DMSetBasicAdjacency", adjCone, &adjCone, &flg));
3499   PetscCall(PetscOptionsBool("-dm_plex_adj_closure", "Set adjacency size", "DMSetBasicAdjacency", adjClosure, &adjClosure, &flg2));
3500   if (flg || flg2) PetscCall(DMSetBasicAdjacency(dm, adjCone, adjClosure));
3501 
3502   switch (cell) {
3503   case DM_POLYTOPE_POINT:
3504   case DM_POLYTOPE_SEGMENT:
3505   case DM_POLYTOPE_POINT_PRISM_TENSOR:
3506   case DM_POLYTOPE_TRIANGLE:
3507   case DM_POLYTOPE_QUADRILATERAL:
3508   case DM_POLYTOPE_TETRAHEDRON:
3509   case DM_POLYTOPE_HEXAHEDRON: *useCoordSpace = PETSC_TRUE; break;
3510   default: *useCoordSpace = PETSC_FALSE; break;
3511   }
3512 
3513   if (fflg) {
3514     DM dmnew;
3515 
3516     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), filename, plexname, interpolate, &dmnew));
3517     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3518     PetscCall(DMPlexReplace_Static(dm, &dmnew));
3519   } else if (refDomain) {
3520     PetscCall(DMPlexCreateReferenceCell_Internal(dm, cell));
3521   } else if (bdfflg) {
3522     DM bdm, dmnew;
3523 
3524     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), bdFilename, plexname, interpolate, &bdm));
3525     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)bdm, "bd_"));
3526     PetscCall(DMSetFromOptions(bdm));
3527     PetscCall(DMPlexGenerate(bdm, NULL, interpolate, &dmnew));
3528     PetscCall(DMDestroy(&bdm));
3529     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3530     PetscCall(DMPlexReplace_Static(dm, &dmnew));
3531   } else {
3532     PetscCall(PetscObjectSetName((PetscObject)dm, DMPlexShapes[shape]));
3533     switch (shape) {
3534     case DM_SHAPE_BOX: {
3535       PetscInt       faces[3] = {0, 0, 0};
3536       PetscReal      lower[3] = {0, 0, 0};
3537       PetscReal      upper[3] = {1, 1, 1};
3538       DMBoundaryType bdt[3]   = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3539       PetscInt       i, n;
3540 
3541       n = dim;
3542       for (i = 0; i < dim; ++i) faces[i] = (dim == 1 ? 1 : 4 - dim);
3543       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3544       n = 3;
3545       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3546       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3547       n = 3;
3548       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3549       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3550       n = 3;
3551       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
3552       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3553       switch (cell) {
3554       case DM_POLYTOPE_TRI_PRISM_TENSOR:
3555         PetscCall(DMPlexCreateWedgeBoxMesh_Internal(dm, faces, lower, upper, bdt));
3556         if (!interpolate) {
3557           DM udm;
3558 
3559           PetscCall(DMPlexUninterpolate(dm, &udm));
3560           PetscCall(DMPlexReplace_Static(dm, &udm));
3561         }
3562         break;
3563       default: PetscCall(DMPlexCreateBoxMesh_Internal(dm, dim, simplex, faces, lower, upper, bdt, interpolate)); break;
3564       }
3565     } break;
3566     case DM_SHAPE_BOX_SURFACE: {
3567       PetscInt  faces[3] = {0, 0, 0};
3568       PetscReal lower[3] = {0, 0, 0};
3569       PetscReal upper[3] = {1, 1, 1};
3570       PetscInt  i, n;
3571 
3572       n = dim + 1;
3573       for (i = 0; i < dim + 1; ++i) faces[i] = (dim + 1 == 1 ? 1 : 4 - (dim + 1));
3574       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3575       n = 3;
3576       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3577       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3578       n = 3;
3579       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3580       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3581       PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(dm, dim + 1, faces, lower, upper, interpolate));
3582     } break;
3583     case DM_SHAPE_SPHERE: {
3584       PetscReal R = 1.0;
3585 
3586       PetscCall(PetscOptionsReal("-dm_plex_sphere_radius", "Radius of the sphere", "", R, &R, &flg));
3587       PetscCall(DMPlexCreateSphereMesh_Internal(dm, dim, simplex, R));
3588     } break;
3589     case DM_SHAPE_BALL: {
3590       PetscReal R = 1.0;
3591 
3592       PetscCall(PetscOptionsReal("-dm_plex_ball_radius", "Radius of the ball", "", R, &R, &flg));
3593       PetscCall(DMPlexCreateBallMesh_Internal(dm, dim, R));
3594     } break;
3595     case DM_SHAPE_CYLINDER: {
3596       DMBoundaryType bdt = DM_BOUNDARY_NONE;
3597       PetscInt       Nw  = 6;
3598 
3599       PetscCall(PetscOptionsEnum("-dm_plex_cylinder_bd", "Boundary type in the z direction", "", DMBoundaryTypes, (PetscEnum)bdt, (PetscEnum *)&bdt, NULL));
3600       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_wedges", "Number of wedges around the cylinder", "", Nw, &Nw, NULL));
3601       switch (cell) {
3602       case DM_POLYTOPE_TRI_PRISM_TENSOR: PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(dm, Nw, interpolate)); break;
3603       default: PetscCall(DMPlexCreateHexCylinderMesh_Internal(dm, bdt)); break;
3604       }
3605     } break;
3606     case DM_SHAPE_SCHWARZ_P: // fallthrough
3607     case DM_SHAPE_GYROID: {
3608       PetscInt       extent[3] = {1, 1, 1}, refine = 0, layers = 0, three;
3609       PetscReal      thickness   = 0.;
3610       DMBoundaryType periodic[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3611       DMPlexTPSType  tps_type    = shape == DM_SHAPE_SCHWARZ_P ? DMPLEX_TPS_SCHWARZ_P : DMPLEX_TPS_GYROID;
3612       PetscBool      tps_distribute;
3613       PetscCall(PetscOptionsIntArray("-dm_plex_tps_extent", "Number of replicas for each of three dimensions", NULL, extent, (three = 3, &three), NULL));
3614       PetscCall(PetscOptionsInt("-dm_plex_tps_refine", "Number of refinements", NULL, refine, &refine, NULL));
3615       PetscCall(PetscOptionsEnumArray("-dm_plex_tps_periodic", "Periodicity in each of three dimensions", NULL, DMBoundaryTypes, (PetscEnum *)periodic, (three = 3, &three), NULL));
3616       PetscCall(PetscOptionsInt("-dm_plex_tps_layers", "Number of layers in volumetric extrusion (or zero to not extrude)", NULL, layers, &layers, NULL));
3617       PetscCall(PetscOptionsReal("-dm_plex_tps_thickness", "Thickness of volumetric extrusion", NULL, thickness, &thickness, NULL));
3618       PetscCall(DMPlexDistributeGetDefault(dm, &tps_distribute));
3619       PetscCall(PetscOptionsBool("-dm_plex_tps_distribute", "Distribute the 2D mesh prior to refinement and extrusion", NULL, tps_distribute, &tps_distribute, NULL));
3620       PetscCall(DMPlexCreateTPSMesh_Internal(dm, tps_type, extent, periodic, tps_distribute, refine, layers, thickness));
3621     } break;
3622     case DM_SHAPE_DOUBLET: {
3623       DM        dmnew;
3624       PetscReal rl = 0.0;
3625 
3626       PetscCall(PetscOptionsReal("-dm_plex_doublet_refinementlimit", "Refinement limit", NULL, rl, &rl, NULL));
3627       PetscCall(DMPlexCreateDoublet(PetscObjectComm((PetscObject)dm), dim, simplex, interpolate, rl, &dmnew));
3628       PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3629       PetscCall(DMPlexReplace_Static(dm, &dmnew));
3630     } break;
3631     default: SETERRQ(comm, PETSC_ERR_SUP, "Domain shape %s is unsupported", DMPlexShapes[shape]);
3632     }
3633   }
3634   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3635   if (!((PetscObject)dm)->name && nameflg) { PetscCall(PetscObjectSetName((PetscObject)dm, plexname)); }
3636   PetscFunctionReturn(0);
3637 }
3638 
3639 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(DM dm, PetscOptionItems *PetscOptionsObject) {
3640   DM_Plex  *mesh = (DM_Plex *)dm->data;
3641   PetscBool flg, flg2;
3642   char      bdLabel[PETSC_MAX_PATH_LEN];
3643 
3644   PetscFunctionBegin;
3645   /* Handle viewing */
3646   PetscCall(PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL));
3647   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fem", "Debug output level all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL, 0));
3648   PetscCall(PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL));
3649   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL, 0));
3650   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_locate", "Debug output level all point location computations", "DMLocatePoints", 0, &mesh->printLocate, NULL, 0));
3651   PetscCall(DMMonitorSetFromOptions(dm, "-dm_plex_monitor_throughput", "Monitor the simulation throughput", "DMPlexMonitorThroughput", DMPlexMonitorThroughput, NULL, &flg));
3652   if (flg) PetscCall(PetscLogDefaultBegin());
3653   /* Labeling */
3654   PetscCall(PetscOptionsString("-dm_plex_boundary_label", "Label to mark the mesh boundary", "", bdLabel, bdLabel, sizeof(bdLabel), &flg));
3655   if (flg) PetscCall(DMPlexCreateBoundaryLabel_Private(dm, bdLabel));
3656   /* Point Location */
3657   PetscCall(PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL));
3658   /* Partitioning and distribution */
3659   PetscCall(PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL));
3660   /* Generation and remeshing */
3661   PetscCall(PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL));
3662   /* Projection behavior */
3663   PetscCall(PetscOptionsBoundedInt("-dm_plex_max_projection_height", "Maxmimum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL, 0));
3664   PetscCall(PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL));
3665   /* Checking structure */
3666   {
3667     PetscBool all = PETSC_FALSE;
3668 
3669     PetscCall(PetscOptionsBool("-dm_plex_check_all", "Perform all basic checks", "DMPlexCheck", PETSC_FALSE, &all, NULL));
3670     if (all) {
3671       PetscCall(DMPlexCheck(dm));
3672     } else {
3673       PetscCall(PetscOptionsBool("-dm_plex_check_symmetry", "Check that the adjacency information in the mesh is symmetric", "DMPlexCheckSymmetry", PETSC_FALSE, &flg, &flg2));
3674       if (flg && flg2) PetscCall(DMPlexCheckSymmetry(dm));
3675       PetscCall(PetscOptionsBool("-dm_plex_check_skeleton", "Check that each cell has the correct number of vertices (only for homogeneous simplex or tensor meshes)", "DMPlexCheckSkeleton", PETSC_FALSE, &flg, &flg2));
3676       if (flg && flg2) PetscCall(DMPlexCheckSkeleton(dm, 0));
3677       PetscCall(PetscOptionsBool("-dm_plex_check_faces", "Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type", "DMPlexCheckFaces", PETSC_FALSE, &flg, &flg2));
3678       if (flg && flg2) PetscCall(DMPlexCheckFaces(dm, 0));
3679       PetscCall(PetscOptionsBool("-dm_plex_check_geometry", "Check that cells have positive volume", "DMPlexCheckGeometry", PETSC_FALSE, &flg, &flg2));
3680       if (flg && flg2) PetscCall(DMPlexCheckGeometry(dm));
3681       PetscCall(PetscOptionsBool("-dm_plex_check_pointsf", "Check some necessary conditions for PointSF", "DMPlexCheckPointSF", PETSC_FALSE, &flg, &flg2));
3682       if (flg && flg2) PetscCall(DMPlexCheckPointSF(dm, NULL));
3683       PetscCall(PetscOptionsBool("-dm_plex_check_interface_cones", "Check points on inter-partition interfaces have conforming order of cone points", "DMPlexCheckInterfaceCones", PETSC_FALSE, &flg, &flg2));
3684       if (flg && flg2) PetscCall(DMPlexCheckInterfaceCones(dm));
3685     }
3686     PetscCall(PetscOptionsBool("-dm_plex_check_cell_shape", "Check cell shape", "DMPlexCheckCellShape", PETSC_FALSE, &flg, &flg2));
3687     if (flg && flg2) PetscCall(DMPlexCheckCellShape(dm, PETSC_TRUE, PETSC_DETERMINE));
3688   }
3689   {
3690     PetscReal scale = 1.0;
3691 
3692     PetscCall(PetscOptionsReal("-dm_plex_scale", "Scale factor for mesh coordinates", "DMPlexScale", scale, &scale, &flg));
3693     if (flg) {
3694       Vec coordinates, coordinatesLocal;
3695 
3696       PetscCall(DMGetCoordinates(dm, &coordinates));
3697       PetscCall(DMGetCoordinatesLocal(dm, &coordinatesLocal));
3698       PetscCall(VecScale(coordinates, scale));
3699       PetscCall(VecScale(coordinatesLocal, scale));
3700     }
3701   }
3702   PetscCall(PetscPartitionerSetFromOptions(mesh->partitioner));
3703   PetscFunctionReturn(0);
3704 }
3705 
3706 PetscErrorCode DMSetFromOptions_Overlap_Plex(DM dm, PetscOptionItems *PetscOptionsObject, PetscInt *overlap) {
3707   PetscInt  numOvLabels = 16, numOvExLabels = 16;
3708   char     *ovLabelNames[16], *ovExLabelNames[16];
3709   PetscInt  numOvValues = 16, numOvExValues = 16, l;
3710   PetscBool flg;
3711 
3712   PetscFunctionBegin;
3713   PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", *overlap, overlap, NULL, 0));
3714   PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_labels", "List of overlap label names", "DMPlexDistribute", ovLabelNames, &numOvLabels, &flg));
3715   if (!flg) numOvLabels = 0;
3716   if (numOvLabels) {
3717     ((DM_Plex *)dm->data)->numOvLabels = numOvLabels;
3718     for (l = 0; l < numOvLabels; ++l) {
3719       PetscCall(DMGetLabel(dm, ovLabelNames[l], &((DM_Plex *)dm->data)->ovLabels[l]));
3720       PetscCheck(((DM_Plex *)dm->data)->ovLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovLabelNames[l]);
3721       PetscCall(PetscFree(ovLabelNames[l]));
3722     }
3723     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_values", "List of overlap label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovValues, &numOvValues, &flg));
3724     if (!flg) numOvValues = 0;
3725     PetscCheck(numOvLabels == numOvValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvLabels, numOvValues);
3726 
3727     PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_exclude_labels", "List of overlap exclude label names", "DMPlexDistribute", ovExLabelNames, &numOvExLabels, &flg));
3728     if (!flg) numOvExLabels = 0;
3729     ((DM_Plex *)dm->data)->numOvExLabels = numOvExLabels;
3730     for (l = 0; l < numOvExLabels; ++l) {
3731       PetscCall(DMGetLabel(dm, ovExLabelNames[l], &((DM_Plex *)dm->data)->ovExLabels[l]));
3732       PetscCheck(((DM_Plex *)dm->data)->ovExLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovExLabelNames[l]);
3733       PetscCall(PetscFree(ovExLabelNames[l]));
3734     }
3735     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_exclude_values", "List of overlap exclude label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovExValues, &numOvExValues, &flg));
3736     if (!flg) numOvExValues = 0;
3737     PetscCheck(numOvExLabels == numOvExValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of exclude labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvExLabels, numOvExValues);
3738   }
3739   PetscFunctionReturn(0);
3740 }
3741 
3742 static PetscErrorCode DMSetFromOptions_Plex(DM dm, PetscOptionItems *PetscOptionsObject) {
3743   PetscFunctionList        ordlist;
3744   char                     oname[256];
3745   PetscReal                volume    = -1.0;
3746   PetscInt                 prerefine = 0, refine = 0, r, coarsen = 0, overlap = 0, extLayers = 0, dim;
3747   PetscBool                uniformOrig, created = PETSC_FALSE, uniform = PETSC_TRUE, distribute, interpolate = PETSC_TRUE, coordSpace = PETSC_TRUE, remap = PETSC_TRUE, ghostCells = PETSC_FALSE, isHierarchy, ignoreModel = PETSC_FALSE, flg;
3748   DMPlexReorderDefaultFlag reorder;
3749 
3750   PetscFunctionBegin;
3751   PetscOptionsHeadBegin(PetscOptionsObject, "DMPlex Options");
3752   /* Handle automatic creation */
3753   PetscCall(DMGetDimension(dm, &dim));
3754   if (dim < 0) {
3755     PetscCall(DMPlexCreateFromOptions_Internal(PetscOptionsObject, &coordSpace, dm));
3756     created = PETSC_TRUE;
3757   }
3758   PetscCall(DMGetDimension(dm, &dim));
3759   /* Handle interpolation before distribution */
3760   PetscCall(PetscOptionsBool("-dm_plex_interpolate_pre", "Flag to interpolate mesh before distribution", "", interpolate, &interpolate, &flg));
3761   if (flg) {
3762     DMPlexInterpolatedFlag interpolated;
3763 
3764     PetscCall(DMPlexIsInterpolated(dm, &interpolated));
3765     if (interpolated == DMPLEX_INTERPOLATED_FULL && !interpolate) {
3766       DM udm;
3767 
3768       PetscCall(DMPlexUninterpolate(dm, &udm));
3769       PetscCall(DMPlexReplace_Static(dm, &udm));
3770     } else if (interpolated != DMPLEX_INTERPOLATED_FULL && interpolate) {
3771       DM idm;
3772 
3773       PetscCall(DMPlexInterpolate(dm, &idm));
3774       PetscCall(DMPlexReplace_Static(dm, &idm));
3775     }
3776   }
3777   /* Handle DMPlex refinement before distribution */
3778   PetscCall(PetscOptionsBool("-dm_refine_ignore_model", "Flag to ignore the geometry model when refining", "DMCreate", ignoreModel, &ignoreModel, &flg));
3779   if (flg) { ((DM_Plex *)dm->data)->ignoreModel = ignoreModel; }
3780   PetscCall(DMPlexGetRefinementUniform(dm, &uniformOrig));
3781   PetscCall(PetscOptionsBoundedInt("-dm_refine_pre", "The number of refinements before distribution", "DMCreate", prerefine, &prerefine, NULL, 0));
3782   PetscCall(PetscOptionsBool("-dm_refine_remap_pre", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
3783   PetscCall(PetscOptionsBool("-dm_refine_uniform_pre", "Flag for uniform refinement before distribution", "DMCreate", uniform, &uniform, &flg));
3784   if (flg) PetscCall(DMPlexSetRefinementUniform(dm, uniform));
3785   PetscCall(PetscOptionsReal("-dm_refine_volume_limit_pre", "The maximum cell volume after refinement before distribution", "DMCreate", volume, &volume, &flg));
3786   if (flg) {
3787     PetscCall(DMPlexSetRefinementUniform(dm, PETSC_FALSE));
3788     PetscCall(DMPlexSetRefinementLimit(dm, volume));
3789     prerefine = PetscMax(prerefine, 1);
3790   }
3791   for (r = 0; r < prerefine; ++r) {
3792     DM             rdm;
3793     PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
3794 
3795     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3796     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
3797     PetscCall(DMPlexReplace_Static(dm, &rdm));
3798     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3799     if (coordFunc && remap) {
3800       PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
3801       ((DM_Plex *)dm->data)->coordFunc = coordFunc;
3802     }
3803   }
3804   PetscCall(DMPlexSetRefinementUniform(dm, uniformOrig));
3805   /* Handle DMPlex extrusion before distribution */
3806   PetscCall(PetscOptionsBoundedInt("-dm_extrude", "The number of layers to extrude", "", extLayers, &extLayers, NULL, 0));
3807   if (extLayers) {
3808     DM edm;
3809 
3810     PetscCall(DMExtrude(dm, extLayers, &edm));
3811     PetscCall(DMPlexReplace_Static(dm, &edm));
3812     ((DM_Plex *)dm->data)->coordFunc = NULL;
3813     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3814     extLayers = 0;
3815   }
3816   /* Handle DMPlex reordering before distribution */
3817   PetscCall(DMPlexReorderGetDefault(dm, &reorder));
3818   PetscCall(MatGetOrderingList(&ordlist));
3819   PetscCall(PetscStrncpy(oname, MATORDERINGNATURAL, sizeof(oname)));
3820   PetscCall(PetscOptionsFList("-dm_plex_reorder", "Set mesh reordering type", "DMPlexGetOrdering", ordlist, MATORDERINGNATURAL, oname, sizeof(oname), &flg));
3821   if (reorder == DMPLEX_REORDER_DEFAULT_TRUE || flg) {
3822     DM pdm;
3823     IS perm;
3824 
3825     PetscCall(DMPlexGetOrdering(dm, oname, NULL, &perm));
3826     PetscCall(DMPlexPermute(dm, perm, &pdm));
3827     PetscCall(ISDestroy(&perm));
3828     PetscCall(DMPlexReplace_Static(dm, &pdm));
3829     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3830   }
3831   /* Handle DMPlex distribution */
3832   PetscCall(DMPlexDistributeGetDefault(dm, &distribute));
3833   PetscCall(PetscOptionsBool("-dm_distribute", "Flag to redistribute a mesh among processes", "DMPlexDistribute", distribute, &distribute, NULL));
3834   PetscCall(DMSetFromOptions_Overlap_Plex(dm, PetscOptionsObject, &overlap));
3835   if (distribute) {
3836     DM               pdm = NULL;
3837     PetscPartitioner part;
3838 
3839     PetscCall(DMPlexGetPartitioner(dm, &part));
3840     PetscCall(PetscPartitionerSetFromOptions(part));
3841     PetscCall(DMPlexDistribute(dm, overlap, NULL, &pdm));
3842     if (pdm) { PetscCall(DMPlexReplace_Static(dm, &pdm)); }
3843   }
3844   /* Create coordinate space */
3845   if (created) {
3846     DM_Plex  *mesh   = (DM_Plex *)dm->data;
3847     PetscInt  degree = 1;
3848     PetscBool flg;
3849 
3850     PetscCall(PetscOptionsBool("-dm_coord_space", "Use an FEM space for coordinates", "", coordSpace, &coordSpace, &flg));
3851     PetscCall(PetscOptionsInt("-dm_coord_petscspace_degree", "FEM degree for coordinate space", "", degree, &degree, NULL));
3852     if (coordSpace) PetscCall(DMPlexCreateCoordinateSpace(dm, degree, mesh->coordFunc));
3853     if (flg && !coordSpace) {
3854       DM           cdm;
3855       PetscDS      cds;
3856       PetscObject  obj;
3857       PetscClassId id;
3858 
3859       PetscCall(DMGetCoordinateDM(dm, &cdm));
3860       PetscCall(DMGetDS(cdm, &cds));
3861       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
3862       PetscCall(PetscObjectGetClassId(obj, &id));
3863       if (id == PETSCFE_CLASSID) {
3864         PetscContainer dummy;
3865 
3866         PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &dummy));
3867         PetscCall(PetscObjectSetName((PetscObject)dummy, "coordinates"));
3868         PetscCall(DMSetField(cdm, 0, NULL, (PetscObject)dummy));
3869         PetscCall(PetscContainerDestroy(&dummy));
3870         PetscCall(DMClearDS(cdm));
3871       }
3872       mesh->coordFunc = NULL;
3873     }
3874     PetscCall(PetscOptionsBool("-dm_sparse_localize", "Localize only necessary cells", "", dm->sparseLocalize, &dm->sparseLocalize, &flg));
3875     PetscCall(DMLocalizeCoordinates(dm));
3876   }
3877   /* Handle DMPlex refinement */
3878   remap = PETSC_TRUE;
3879   PetscCall(PetscOptionsBoundedInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL, 0));
3880   PetscCall(PetscOptionsBool("-dm_refine_remap", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
3881   PetscCall(PetscOptionsBoundedInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy, 0));
3882   if (refine) PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3883   if (refine && isHierarchy) {
3884     DM *dms, coarseDM;
3885 
3886     PetscCall(DMGetCoarseDM(dm, &coarseDM));
3887     PetscCall(PetscObjectReference((PetscObject)coarseDM));
3888     PetscCall(PetscMalloc1(refine, &dms));
3889     PetscCall(DMRefineHierarchy(dm, refine, dms));
3890     /* Total hack since we do not pass in a pointer */
3891     PetscCall(DMPlexSwap_Static(dm, dms[refine - 1]));
3892     if (refine == 1) {
3893       PetscCall(DMSetCoarseDM(dm, dms[0]));
3894       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
3895     } else {
3896       PetscCall(DMSetCoarseDM(dm, dms[refine - 2]));
3897       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
3898       PetscCall(DMSetCoarseDM(dms[0], dms[refine - 1]));
3899       PetscCall(DMPlexSetRegularRefinement(dms[0], PETSC_TRUE));
3900     }
3901     PetscCall(DMSetCoarseDM(dms[refine - 1], coarseDM));
3902     PetscCall(PetscObjectDereference((PetscObject)coarseDM));
3903     /* Free DMs */
3904     for (r = 0; r < refine; ++r) {
3905       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
3906       PetscCall(DMDestroy(&dms[r]));
3907     }
3908     PetscCall(PetscFree(dms));
3909   } else {
3910     for (r = 0; r < refine; ++r) {
3911       DM             rdm;
3912       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
3913 
3914       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3915       PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
3916       /* Total hack since we do not pass in a pointer */
3917       PetscCall(DMPlexReplace_Static(dm, &rdm));
3918       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3919       if (coordFunc && remap) {
3920         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
3921         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
3922       }
3923     }
3924   }
3925   /* Handle DMPlex coarsening */
3926   PetscCall(PetscOptionsBoundedInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL, 0));
3927   PetscCall(PetscOptionsBoundedInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy, 0));
3928   if (coarsen && isHierarchy) {
3929     DM *dms;
3930 
3931     PetscCall(PetscMalloc1(coarsen, &dms));
3932     PetscCall(DMCoarsenHierarchy(dm, coarsen, dms));
3933     /* Free DMs */
3934     for (r = 0; r < coarsen; ++r) {
3935       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
3936       PetscCall(DMDestroy(&dms[r]));
3937     }
3938     PetscCall(PetscFree(dms));
3939   } else {
3940     for (r = 0; r < coarsen; ++r) {
3941       DM             cdm;
3942       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
3943 
3944       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3945       PetscCall(DMCoarsen(dm, PetscObjectComm((PetscObject)dm), &cdm));
3946       /* Total hack since we do not pass in a pointer */
3947       PetscCall(DMPlexReplace_Static(dm, &cdm));
3948       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3949       if (coordFunc) {
3950         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
3951         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
3952       }
3953     }
3954   }
3955   /* Handle ghost cells */
3956   PetscCall(PetscOptionsBool("-dm_plex_create_fv_ghost_cells", "Flag to create finite volume ghost cells on the boundary", "DMCreate", ghostCells, &ghostCells, NULL));
3957   if (ghostCells) {
3958     DM   gdm;
3959     char lname[PETSC_MAX_PATH_LEN];
3960 
3961     lname[0] = '\0';
3962     PetscCall(PetscOptionsString("-dm_plex_fv_ghost_cells_label", "Label name for ghost cells boundary", "DMCreate", lname, lname, sizeof(lname), &flg));
3963     PetscCall(DMPlexConstructGhostCells(dm, flg ? lname : NULL, NULL, &gdm));
3964     PetscCall(DMPlexReplace_Static(dm, &gdm));
3965   }
3966   /* Handle 1D order */
3967   if (reorder != DMPLEX_REORDER_DEFAULT_FALSE && dim == 1) {
3968     DM           cdm, rdm;
3969     PetscDS      cds;
3970     PetscObject  obj;
3971     PetscClassId id = PETSC_OBJECT_CLASSID;
3972     IS           perm;
3973     PetscInt     Nf;
3974     PetscBool    distributed;
3975 
3976     PetscCall(DMPlexIsDistributed(dm, &distributed));
3977     PetscCall(DMGetCoordinateDM(dm, &cdm));
3978     PetscCall(DMGetDS(cdm, &cds));
3979     PetscCall(PetscDSGetNumFields(cds, &Nf));
3980     if (Nf) {
3981       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
3982       PetscCall(PetscObjectGetClassId(obj, &id));
3983     }
3984     if (!distributed && id != PETSCFE_CLASSID) {
3985       PetscCall(DMPlexGetOrdering1D(dm, &perm));
3986       PetscCall(DMPlexPermute(dm, perm, &rdm));
3987       PetscCall(DMPlexReplace_Static(dm, &rdm));
3988       PetscCall(ISDestroy(&perm));
3989     }
3990   }
3991   /* Handle */
3992   PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3993   PetscOptionsHeadEnd();
3994   PetscFunctionReturn(0);
3995 }
3996 
3997 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm, Vec *vec) {
3998   PetscFunctionBegin;
3999   PetscCall(DMCreateGlobalVector_Section_Private(dm, vec));
4000   /* PetscCall(VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM)); */
4001   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex));
4002   PetscCall(VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void))VecView_Plex_Native));
4003   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex));
4004   PetscCall(VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void))VecLoad_Plex_Native));
4005   PetscFunctionReturn(0);
4006 }
4007 
4008 static PetscErrorCode DMCreateLocalVector_Plex(DM dm, Vec *vec) {
4009   PetscFunctionBegin;
4010   PetscCall(DMCreateLocalVector_Section_Private(dm, vec));
4011   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex_Local));
4012   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex_Local));
4013   PetscFunctionReturn(0);
4014 }
4015 
4016 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd) {
4017   PetscInt depth, d;
4018 
4019   PetscFunctionBegin;
4020   PetscCall(DMPlexGetDepth(dm, &depth));
4021   if (depth == 1) {
4022     PetscCall(DMGetDimension(dm, &d));
4023     if (dim == 0) PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4024     else if (dim == d) PetscCall(DMPlexGetDepthStratum(dm, 1, pStart, pEnd));
4025     else {
4026       *pStart = 0;
4027       *pEnd   = 0;
4028     }
4029   } else {
4030     PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4031   }
4032   PetscFunctionReturn(0);
4033 }
4034 
4035 static PetscErrorCode DMGetNeighbors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[]) {
4036   PetscSF            sf;
4037   PetscInt           niranks, njranks, n;
4038   const PetscMPIInt *iranks, *jranks;
4039   DM_Plex           *data = (DM_Plex *)dm->data;
4040 
4041   PetscFunctionBegin;
4042   PetscCall(DMGetPointSF(dm, &sf));
4043   if (!data->neighbors) {
4044     PetscCall(PetscSFSetUp(sf));
4045     PetscCall(PetscSFGetRootRanks(sf, &njranks, &jranks, NULL, NULL, NULL));
4046     PetscCall(PetscSFGetLeafRanks(sf, &niranks, &iranks, NULL, NULL));
4047     PetscCall(PetscMalloc1(njranks + niranks + 1, &data->neighbors));
4048     PetscCall(PetscArraycpy(data->neighbors + 1, jranks, njranks));
4049     PetscCall(PetscArraycpy(data->neighbors + njranks + 1, iranks, niranks));
4050     n = njranks + niranks;
4051     PetscCall(PetscSortRemoveDupsMPIInt(&n, data->neighbors + 1));
4052     /* The following cast should never fail: can't have more neighbors than PETSC_MPI_INT_MAX */
4053     PetscCall(PetscMPIIntCast(n, data->neighbors));
4054   }
4055   if (nranks) *nranks = data->neighbors[0];
4056   if (ranks) {
4057     if (data->neighbors[0]) *ranks = data->neighbors + 1;
4058     else *ranks = NULL;
4059   }
4060   PetscFunctionReturn(0);
4061 }
4062 
4063 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM, DM, Mat, Vec, Vec);
4064 
4065 static PetscErrorCode DMInitialize_Plex(DM dm) {
4066   PetscFunctionBegin;
4067   dm->ops->view                      = DMView_Plex;
4068   dm->ops->load                      = DMLoad_Plex;
4069   dm->ops->setfromoptions            = DMSetFromOptions_Plex;
4070   dm->ops->clone                     = DMClone_Plex;
4071   dm->ops->setup                     = DMSetUp_Plex;
4072   dm->ops->createlocalsection        = DMCreateLocalSection_Plex;
4073   dm->ops->createdefaultconstraints  = DMCreateDefaultConstraints_Plex;
4074   dm->ops->createglobalvector        = DMCreateGlobalVector_Plex;
4075   dm->ops->createlocalvector         = DMCreateLocalVector_Plex;
4076   dm->ops->getlocaltoglobalmapping   = NULL;
4077   dm->ops->createfieldis             = NULL;
4078   dm->ops->createcoordinatedm        = DMCreateCoordinateDM_Plex;
4079   dm->ops->createcoordinatefield     = DMCreateCoordinateField_Plex;
4080   dm->ops->getcoloring               = NULL;
4081   dm->ops->creatematrix              = DMCreateMatrix_Plex;
4082   dm->ops->createinterpolation       = DMCreateInterpolation_Plex;
4083   dm->ops->createmassmatrix          = DMCreateMassMatrix_Plex;
4084   dm->ops->createmassmatrixlumped    = DMCreateMassMatrixLumped_Plex;
4085   dm->ops->createinjection           = DMCreateInjection_Plex;
4086   dm->ops->refine                    = DMRefine_Plex;
4087   dm->ops->coarsen                   = DMCoarsen_Plex;
4088   dm->ops->refinehierarchy           = DMRefineHierarchy_Plex;
4089   dm->ops->coarsenhierarchy          = DMCoarsenHierarchy_Plex;
4090   dm->ops->extrude                   = DMExtrude_Plex;
4091   dm->ops->globaltolocalbegin        = NULL;
4092   dm->ops->globaltolocalend          = NULL;
4093   dm->ops->localtoglobalbegin        = NULL;
4094   dm->ops->localtoglobalend          = NULL;
4095   dm->ops->destroy                   = DMDestroy_Plex;
4096   dm->ops->createsubdm               = DMCreateSubDM_Plex;
4097   dm->ops->createsuperdm             = DMCreateSuperDM_Plex;
4098   dm->ops->getdimpoints              = DMGetDimPoints_Plex;
4099   dm->ops->locatepoints              = DMLocatePoints_Plex;
4100   dm->ops->projectfunctionlocal      = DMProjectFunctionLocal_Plex;
4101   dm->ops->projectfunctionlabellocal = DMProjectFunctionLabelLocal_Plex;
4102   dm->ops->projectfieldlocal         = DMProjectFieldLocal_Plex;
4103   dm->ops->projectfieldlabellocal    = DMProjectFieldLabelLocal_Plex;
4104   dm->ops->projectbdfieldlabellocal  = DMProjectBdFieldLabelLocal_Plex;
4105   dm->ops->computel2diff             = DMComputeL2Diff_Plex;
4106   dm->ops->computel2gradientdiff     = DMComputeL2GradientDiff_Plex;
4107   dm->ops->computel2fielddiff        = DMComputeL2FieldDiff_Plex;
4108   dm->ops->getneighbors              = DMGetNeighbors_Plex;
4109   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertBoundaryValues_C", DMPlexInsertBoundaryValues_Plex));
4110   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertTimeDerviativeBoundaryValues_C", DMPlexInsertTimeDerivativeBoundaryValues_Plex));
4111   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMSetUpGLVisViewer_C", DMSetUpGLVisViewer_Plex));
4112   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMCreateNeumannOverlap_C", DMCreateNeumannOverlap_Plex));
4113   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4114   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeGetDefault_C", DMPlexDistributeGetDefault_Plex));
4115   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeSetDefault_C", DMPlexDistributeSetDefault_Plex));
4116   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderGetDefault_C", DMPlexReorderGetDefault_Plex));
4117   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderSetDefault_C", DMPlexReorderSetDefault_Plex));
4118   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMInterpolateSolution_C", DMInterpolateSolution_Plex));
4119   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4120   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetOverlap_C", DMPlexSetOverlap_Plex));
4121   PetscFunctionReturn(0);
4122 }
4123 
4124 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm) {
4125   DM_Plex *mesh = (DM_Plex *)dm->data;
4126 
4127   PetscFunctionBegin;
4128   mesh->refct++;
4129   (*newdm)->data = mesh;
4130   PetscCall(PetscObjectChangeTypeName((PetscObject)*newdm, DMPLEX));
4131   PetscCall(DMInitialize_Plex(*newdm));
4132   PetscFunctionReturn(0);
4133 }
4134 
4135 /*MC
4136   DMPLEX = "plex" - A DM object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
4137                     In the local representation, Vecs contain all unknowns in the interior and shared boundary. This is
4138                     specified by a PetscSection object. Ownership in the global representation is determined by
4139                     ownership of the underlying DMPlex points. This is specified by another PetscSection object.
4140 
4141   Options Database Keys:
4142 + -dm_refine_pre                     - Refine mesh before distribution
4143 + -dm_refine_uniform_pre             - Choose uniform or generator-based refinement
4144 + -dm_refine_volume_limit_pre        - Cell volume limit after pre-refinement using generator
4145 . -dm_distribute                     - Distribute mesh across processes
4146 . -dm_distribute_overlap             - Number of cells to overlap for distribution
4147 . -dm_refine                         - Refine mesh after distribution
4148 . -dm_plex_hash_location             - Use grid hashing for point location
4149 . -dm_plex_hash_box_faces <n,m,p>    - The number of divisions in each direction of the grid hash
4150 . -dm_plex_partition_balance         - Attempt to evenly divide points on partition boundary between processes
4151 . -dm_plex_remesh_bd                 - Allow changes to the boundary on remeshing
4152 . -dm_plex_max_projection_height     - Maxmimum mesh point height used to project locally
4153 . -dm_plex_regular_refinement        - Use special nested projection algorithm for regular refinement
4154 . -dm_plex_check_all                 - Perform all shecks below
4155 . -dm_plex_check_symmetry            - Check that the adjacency information in the mesh is symmetric
4156 . -dm_plex_check_skeleton <celltype> - Check that each cell has the correct number of vertices
4157 . -dm_plex_check_faces <celltype>    - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
4158 . -dm_plex_check_geometry            - Check that cells have positive volume
4159 . -dm_view :mesh.tex:ascii_latex     - View the mesh in LaTeX/TikZ
4160 . -dm_plex_view_scale <num>          - Scale the TikZ
4161 - -dm_plex_print_fem <num>           - View FEM assembly information, such as element vectors and matrices
4162 
4163   Level: intermediate
4164 
4165 .seealso: `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`
4166 M*/
4167 
4168 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm) {
4169   DM_Plex *mesh;
4170   PetscInt unit;
4171 
4172   PetscFunctionBegin;
4173   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4174   PetscCall(PetscNewLog(dm, &mesh));
4175   dm->data = mesh;
4176 
4177   mesh->refct = 1;
4178   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection));
4179   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection));
4180   mesh->refinementUniform      = PETSC_TRUE;
4181   mesh->refinementLimit        = -1.0;
4182   mesh->distDefault            = PETSC_TRUE;
4183   mesh->reorderDefault         = DMPLEX_REORDER_DEFAULT_NOTSET;
4184   mesh->interpolated           = DMPLEX_INTERPOLATED_INVALID;
4185   mesh->interpolatedCollective = DMPLEX_INTERPOLATED_INVALID;
4186 
4187   PetscCall(PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner));
4188   mesh->remeshBd = PETSC_FALSE;
4189 
4190   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
4191 
4192   mesh->depthState    = -1;
4193   mesh->celltypeState = -1;
4194   mesh->printTol      = 1.0e-10;
4195 
4196   PetscCall(DMInitialize_Plex(dm));
4197   PetscFunctionReturn(0);
4198 }
4199 
4200 /*@
4201   DMPlexCreate - Creates a DMPlex object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
4202 
4203   Collective
4204 
4205   Input Parameter:
4206 . comm - The communicator for the DMPlex object
4207 
4208   Output Parameter:
4209 . mesh  - The DMPlex object
4210 
4211   Level: beginner
4212 
4213 @*/
4214 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh) {
4215   PetscFunctionBegin;
4216   PetscValidPointer(mesh, 2);
4217   PetscCall(DMCreate(comm, mesh));
4218   PetscCall(DMSetType(*mesh, DMPLEX));
4219   PetscFunctionReturn(0);
4220 }
4221 
4222 /*@C
4223   DMPlexBuildFromCellListParallel - Build distributed DMPLEX topology from a list of vertices for each cell (common mesh generator output)
4224 
4225   Input Parameters:
4226 + dm - The DM
4227 . numCells - The number of cells owned by this process
4228 . numVertices - The number of vertices to be owned by this process, or PETSC_DECIDE
4229 . NVertices - The global number of vertices, or PETSC_DETERMINE
4230 . numCorners - The number of vertices for each cell
4231 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4232 
4233   Output Parameters:
4234 + vertexSF - (Optional) SF describing complete vertex ownership
4235 - verticesAdjSaved - (Optional) vertex adjacency array
4236 
4237   Notes:
4238   Two triangles sharing a face
4239 $
4240 $        2
4241 $      / | \
4242 $     /  |  \
4243 $    /   |   \
4244 $   0  0 | 1  3
4245 $    \   |   /
4246 $     \  |  /
4247 $      \ | /
4248 $        1
4249 would have input
4250 $  numCells = 2, numVertices = 4
4251 $  cells = [0 1 2  1 3 2]
4252 $
4253 which would result in the DMPlex
4254 $
4255 $        4
4256 $      / | \
4257 $     /  |  \
4258 $    /   |   \
4259 $   2  0 | 1  5
4260 $    \   |   /
4261 $     \  |  /
4262 $      \ | /
4263 $        3
4264 
4265   Vertices are implicitly numbered consecutively 0,...,NVertices.
4266   Each rank owns a chunk of numVertices consecutive vertices.
4267   If numVertices is PETSC_DECIDE, PETSc will distribute them as evenly as possible using PetscLayout.
4268   If NVertices is PETSC_DETERMINE and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
4269   If only NVertices is PETSC_DETERMINE, it is computed as the sum of numVertices over all ranks.
4270 
4271   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
4272 
4273   Not currently supported in Fortran.
4274 
4275   Level: advanced
4276 
4277 .seealso: `DMPlexBuildFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildCoordinatesFromCellListParallel()`
4278 @*/
4279 PetscErrorCode DMPlexBuildFromCellListParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, const PetscInt cells[], PetscSF *vertexSF, PetscInt **verticesAdjSaved) {
4280   PetscSF     sfPoint;
4281   PetscLayout layout;
4282   PetscInt    numVerticesAdj, *verticesAdj, *cones, c, p;
4283 
4284   PetscFunctionBegin;
4285   PetscValidLogicalCollectiveInt(dm, NVertices, 4);
4286   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4287   /* Get/check global number of vertices */
4288   {
4289     PetscInt       NVerticesInCells, i;
4290     const PetscInt len = numCells * numCorners;
4291 
4292     /* NVerticesInCells = max(cells) + 1 */
4293     NVerticesInCells = PETSC_MIN_INT;
4294     for (i = 0; i < len; i++)
4295       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4296     ++NVerticesInCells;
4297     PetscCallMPI(MPI_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
4298 
4299     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
4300     else
4301       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
4302   }
4303   /* Count locally unique vertices */
4304   {
4305     PetscHSetI vhash;
4306     PetscInt   off = 0;
4307 
4308     PetscCall(PetscHSetICreate(&vhash));
4309     for (c = 0; c < numCells; ++c) {
4310       for (p = 0; p < numCorners; ++p) { PetscCall(PetscHSetIAdd(vhash, cells[c * numCorners + p])); }
4311     }
4312     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
4313     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
4314     else { verticesAdj = *verticesAdjSaved; }
4315     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
4316     PetscCall(PetscHSetIDestroy(&vhash));
4317     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
4318   }
4319   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
4320   /* Create cones */
4321   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
4322   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
4323   PetscCall(DMSetUp(dm));
4324   PetscCall(DMPlexGetCones(dm, &cones));
4325   for (c = 0; c < numCells; ++c) {
4326     for (p = 0; p < numCorners; ++p) {
4327       const PetscInt gv = cells[c * numCorners + p];
4328       PetscInt       lv;
4329 
4330       /* Positions within verticesAdj form 0-based local vertex numbering;
4331          we need to shift it by numCells to get correct DAG points (cells go first) */
4332       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
4333       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
4334       cones[c * numCorners + p] = lv + numCells;
4335     }
4336   }
4337   /* Build point sf */
4338   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
4339   PetscCall(PetscLayoutSetSize(layout, NVertices));
4340   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
4341   PetscCall(PetscLayoutSetBlockSize(layout, 1));
4342   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
4343   PetscCall(PetscLayoutDestroy(&layout));
4344   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
4345   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
4346   if (dm->sf) {
4347     const char *prefix;
4348 
4349     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
4350     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
4351   }
4352   PetscCall(DMSetPointSF(dm, sfPoint));
4353   PetscCall(PetscSFDestroy(&sfPoint));
4354   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)(*vertexSF), "Vertex Ownership SF"));
4355   /* Fill in the rest of the topology structure */
4356   PetscCall(DMPlexSymmetrize(dm));
4357   PetscCall(DMPlexStratify(dm));
4358   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4359   PetscFunctionReturn(0);
4360 }
4361 
4362 /*@C
4363   DMPlexBuildCoordinatesFromCellListParallel - Build DM coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4364 
4365   Input Parameters:
4366 + dm - The DM
4367 . spaceDim - The spatial dimension used for coordinates
4368 . sfVert - SF describing complete vertex ownership
4369 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4370 
4371   Level: advanced
4372 
4373   Notes:
4374   Not currently supported in Fortran.
4375 
4376 .seealso: `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellListParallel()`
4377 @*/
4378 PetscErrorCode DMPlexBuildCoordinatesFromCellListParallel(DM dm, PetscInt spaceDim, PetscSF sfVert, const PetscReal vertexCoords[]) {
4379   PetscSection coordSection;
4380   Vec          coordinates;
4381   PetscScalar *coords;
4382   PetscInt     numVertices, numVerticesAdj, coordSize, v, vStart, vEnd;
4383 
4384   PetscFunctionBegin;
4385   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4386   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
4387   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4388   PetscCall(DMSetCoordinateDim(dm, spaceDim));
4389   PetscCall(PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL));
4390   PetscCheck(vEnd - vStart == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Supplied sfVert has wrong number of leaves = %" PetscInt_FMT " != %" PetscInt_FMT " = vEnd - vStart", numVerticesAdj, vEnd - vStart);
4391   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4392   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4393   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
4394   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
4395   for (v = vStart; v < vEnd; ++v) {
4396     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
4397     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
4398   }
4399   PetscCall(PetscSectionSetUp(coordSection));
4400   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4401   PetscCall(VecCreate(PetscObjectComm((PetscObject)dm), &coordinates));
4402   PetscCall(VecSetBlockSize(coordinates, spaceDim));
4403   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4404   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4405   PetscCall(VecSetType(coordinates, VECSTANDARD));
4406   PetscCall(VecGetArray(coordinates, &coords));
4407   {
4408     MPI_Datatype coordtype;
4409 
4410     /* Need a temp buffer for coords if we have complex/single */
4411     PetscCallMPI(MPI_Type_contiguous(spaceDim, MPIU_SCALAR, &coordtype));
4412     PetscCallMPI(MPI_Type_commit(&coordtype));
4413 #if defined(PETSC_USE_COMPLEX)
4414     {
4415       PetscScalar *svertexCoords;
4416       PetscInt     i;
4417       PetscCall(PetscMalloc1(numVertices * spaceDim, &svertexCoords));
4418       for (i = 0; i < numVertices * spaceDim; i++) svertexCoords[i] = vertexCoords[i];
4419       PetscCall(PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4420       PetscCall(PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4421       PetscCall(PetscFree(svertexCoords));
4422     }
4423 #else
4424     PetscCall(PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4425     PetscCall(PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4426 #endif
4427     PetscCallMPI(MPI_Type_free(&coordtype));
4428   }
4429   PetscCall(VecRestoreArray(coordinates, &coords));
4430   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4431   PetscCall(VecDestroy(&coordinates));
4432   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4433   PetscFunctionReturn(0);
4434 }
4435 
4436 /*@
4437   DMPlexCreateFromCellListParallelPetsc - Create distributed DMPLEX from a list of vertices for each cell (common mesh generator output)
4438 
4439   Input Parameters:
4440 + comm - The communicator
4441 . dim - The topological dimension of the mesh
4442 . numCells - The number of cells owned by this process
4443 . numVertices - The number of vertices owned by this process, or PETSC_DECIDE
4444 . NVertices - The global number of vertices, or PETSC_DECIDE
4445 . numCorners - The number of vertices for each cell
4446 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4447 . cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4448 . spaceDim - The spatial dimension used for coordinates
4449 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4450 
4451   Output Parameters:
4452 + dm - The DM
4453 . vertexSF - (Optional) SF describing complete vertex ownership
4454 - verticesAdjSaved - (Optional) vertex adjacency array
4455 
4456   Notes:
4457   This function is just a convenient sequence of DMCreate(), DMSetType(), DMSetDimension(),
4458   DMPlexBuildFromCellListParallel(), DMPlexInterpolate(), DMPlexBuildCoordinatesFromCellListParallel()
4459 
4460   See DMPlexBuildFromCellListParallel() for an example and details about the topology-related parameters.
4461   See DMPlexBuildCoordinatesFromCellListParallel() for details about the geometry-related parameters.
4462 
4463   Level: intermediate
4464 
4465 .seealso: `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
4466 @*/
4467 PetscErrorCode DMPlexCreateFromCellListParallelPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, PetscInt **verticesAdj, DM *dm) {
4468   PetscSF sfVert;
4469 
4470   PetscFunctionBegin;
4471   PetscCall(DMCreate(comm, dm));
4472   PetscCall(DMSetType(*dm, DMPLEX));
4473   PetscValidLogicalCollectiveInt(*dm, dim, 2);
4474   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
4475   PetscCall(DMSetDimension(*dm, dim));
4476   PetscCall(DMPlexBuildFromCellListParallel(*dm, numCells, numVertices, NVertices, numCorners, cells, &sfVert, verticesAdj));
4477   if (interpolate) {
4478     DM idm;
4479 
4480     PetscCall(DMPlexInterpolate(*dm, &idm));
4481     PetscCall(DMDestroy(dm));
4482     *dm = idm;
4483   }
4484   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
4485   if (vertexSF) *vertexSF = sfVert;
4486   else PetscCall(PetscSFDestroy(&sfVert));
4487   PetscFunctionReturn(0);
4488 }
4489 
4490 /*@C
4491   DMPlexBuildFromCellList - Build DMPLEX topology from a list of vertices for each cell (common mesh generator output)
4492 
4493   Input Parameters:
4494 + dm - The DM
4495 . numCells - The number of cells owned by this process
4496 . numVertices - The number of vertices owned by this process, or PETSC_DETERMINE
4497 . numCorners - The number of vertices for each cell
4498 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4499 
4500   Level: advanced
4501 
4502   Notes:
4503   Two triangles sharing a face
4504 $
4505 $        2
4506 $      / | \
4507 $     /  |  \
4508 $    /   |   \
4509 $   0  0 | 1  3
4510 $    \   |   /
4511 $     \  |  /
4512 $      \ | /
4513 $        1
4514 would have input
4515 $  numCells = 2, numVertices = 4
4516 $  cells = [0 1 2  1 3 2]
4517 $
4518 which would result in the DMPlex
4519 $
4520 $        4
4521 $      / | \
4522 $     /  |  \
4523 $    /   |   \
4524 $   2  0 | 1  5
4525 $    \   |   /
4526 $     \  |  /
4527 $      \ | /
4528 $        3
4529 
4530   If numVertices is PETSC_DETERMINE, it is computed by PETSc as the maximum vertex index in cells + 1.
4531 
4532   Not currently supported in Fortran.
4533 
4534 .seealso: `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListPetsc()`
4535 @*/
4536 PetscErrorCode DMPlexBuildFromCellList(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const PetscInt cells[]) {
4537   PetscInt *cones, c, p, dim;
4538 
4539   PetscFunctionBegin;
4540   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4541   PetscCall(DMGetDimension(dm, &dim));
4542   /* Get/check global number of vertices */
4543   {
4544     PetscInt       NVerticesInCells, i;
4545     const PetscInt len = numCells * numCorners;
4546 
4547     /* NVerticesInCells = max(cells) + 1 */
4548     NVerticesInCells = PETSC_MIN_INT;
4549     for (i = 0; i < len; i++)
4550       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4551     ++NVerticesInCells;
4552 
4553     if (numVertices == PETSC_DECIDE) numVertices = NVerticesInCells;
4554     else
4555       PetscCheck(numVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, numVertices, NVerticesInCells);
4556   }
4557   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
4558   for (c = 0; c < numCells; ++c) { PetscCall(DMPlexSetConeSize(dm, c, numCorners)); }
4559   PetscCall(DMSetUp(dm));
4560   PetscCall(DMPlexGetCones(dm, &cones));
4561   for (c = 0; c < numCells; ++c) {
4562     for (p = 0; p < numCorners; ++p) { cones[c * numCorners + p] = cells[c * numCorners + p] + numCells; }
4563   }
4564   PetscCall(DMPlexSymmetrize(dm));
4565   PetscCall(DMPlexStratify(dm));
4566   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4567   PetscFunctionReturn(0);
4568 }
4569 
4570 /*@C
4571   DMPlexBuildCoordinatesFromCellList - Build DM coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4572 
4573   Input Parameters:
4574 + dm - The DM
4575 . spaceDim - The spatial dimension used for coordinates
4576 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4577 
4578   Level: advanced
4579 
4580   Notes:
4581   Not currently supported in Fortran.
4582 
4583 .seealso: `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellList()`
4584 @*/
4585 PetscErrorCode DMPlexBuildCoordinatesFromCellList(DM dm, PetscInt spaceDim, const PetscReal vertexCoords[]) {
4586   PetscSection coordSection;
4587   Vec          coordinates;
4588   DM           cdm;
4589   PetscScalar *coords;
4590   PetscInt     v, vStart, vEnd, d;
4591 
4592   PetscFunctionBegin;
4593   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4594   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
4595   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4596   PetscCall(DMSetCoordinateDim(dm, spaceDim));
4597   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4598   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4599   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
4600   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
4601   for (v = vStart; v < vEnd; ++v) {
4602     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
4603     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
4604   }
4605   PetscCall(PetscSectionSetUp(coordSection));
4606 
4607   PetscCall(DMGetCoordinateDM(dm, &cdm));
4608   PetscCall(DMCreateLocalVector(cdm, &coordinates));
4609   PetscCall(VecSetBlockSize(coordinates, spaceDim));
4610   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4611   PetscCall(VecGetArrayWrite(coordinates, &coords));
4612   for (v = 0; v < vEnd - vStart; ++v) {
4613     for (d = 0; d < spaceDim; ++d) { coords[v * spaceDim + d] = vertexCoords[v * spaceDim + d]; }
4614   }
4615   PetscCall(VecRestoreArrayWrite(coordinates, &coords));
4616   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4617   PetscCall(VecDestroy(&coordinates));
4618   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4619   PetscFunctionReturn(0);
4620 }
4621 
4622 /*@
4623   DMPlexCreateFromCellListPetsc - Create DMPLEX from a list of vertices for each cell (common mesh generator output), but only process 0 takes in the input
4624 
4625   Collective on comm
4626 
4627   Input Parameters:
4628 + comm - The communicator
4629 . dim - The topological dimension of the mesh
4630 . numCells - The number of cells, only on process 0
4631 . numVertices - The number of vertices owned by this process, or PETSC_DECIDE, only on process 0
4632 . numCorners - The number of vertices for each cell, only on process 0
4633 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4634 . cells - An array of numCells*numCorners numbers, the vertices for each cell, only on process 0
4635 . spaceDim - The spatial dimension used for coordinates
4636 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex, only on process 0
4637 
4638   Output Parameter:
4639 . dm - The DM, which only has points on process 0
4640 
4641   Notes:
4642   This function is just a convenient sequence of DMCreate(), DMSetType(), DMSetDimension(), DMPlexBuildFromCellList(),
4643   DMPlexInterpolate(), DMPlexBuildCoordinatesFromCellList()
4644 
4645   See DMPlexBuildFromCellList() for an example and details about the topology-related parameters.
4646   See DMPlexBuildCoordinatesFromCellList() for details about the geometry-related parameters.
4647   See DMPlexCreateFromCellListParallelPetsc() for parallel input
4648 
4649   Level: intermediate
4650 
4651 .seealso: `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellList()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
4652 @*/
4653 PetscErrorCode DMPlexCreateFromCellListPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], DM *dm) {
4654   PetscMPIInt rank;
4655 
4656   PetscFunctionBegin;
4657   PetscCheck(dim, comm, PETSC_ERR_ARG_OUTOFRANGE, "This is not appropriate for 0-dimensional meshes. Consider either creating the DM using DMPlexCreateFromDAG(), by hand, or using DMSwarm.");
4658   PetscCallMPI(MPI_Comm_rank(comm, &rank));
4659   PetscCall(DMCreate(comm, dm));
4660   PetscCall(DMSetType(*dm, DMPLEX));
4661   PetscCall(DMSetDimension(*dm, dim));
4662   if (rank == 0) PetscCall(DMPlexBuildFromCellList(*dm, numCells, numVertices, numCorners, cells));
4663   else PetscCall(DMPlexBuildFromCellList(*dm, 0, 0, 0, NULL));
4664   if (interpolate) {
4665     DM idm;
4666 
4667     PetscCall(DMPlexInterpolate(*dm, &idm));
4668     PetscCall(DMDestroy(dm));
4669     *dm = idm;
4670   }
4671   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, vertexCoords));
4672   else PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, NULL));
4673   PetscFunctionReturn(0);
4674 }
4675 
4676 /*@
4677   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a DM
4678 
4679   Input Parameters:
4680 + dm - The empty DM object, usually from DMCreate() and DMSetDimension()
4681 . depth - The depth of the DAG
4682 . numPoints - Array of size depth + 1 containing the number of points at each depth
4683 . coneSize - The cone size of each point
4684 . cones - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
4685 . coneOrientations - The orientation of each cone point
4686 - vertexCoords - An array of numPoints[0]*spacedim numbers representing the coordinates of each vertex, with spacedim the value set via DMSetCoordinateDim()
4687 
4688   Output Parameter:
4689 . dm - The DM
4690 
4691   Note: Two triangles sharing a face would have input
4692 $  depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
4693 $  cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
4694 $ vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
4695 $
4696 which would result in the DMPlex
4697 $
4698 $        4
4699 $      / | \
4700 $     /  |  \
4701 $    /   |   \
4702 $   2  0 | 1  5
4703 $    \   |   /
4704 $     \  |  /
4705 $      \ | /
4706 $        3
4707 $
4708 $ Notice that all points are numbered consecutively, unlike DMPlexCreateFromCellListPetsc()
4709 
4710   Level: advanced
4711 
4712 .seealso: `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
4713 @*/
4714 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[]) {
4715   Vec          coordinates;
4716   PetscSection coordSection;
4717   PetscScalar *coords;
4718   PetscInt     coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
4719 
4720   PetscFunctionBegin;
4721   PetscCall(DMGetDimension(dm, &dim));
4722   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
4723   PetscCheck(dimEmbed >= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Embedding dimension %" PetscInt_FMT " cannot be less than intrinsic dimension %" PetscInt_FMT, dimEmbed, dim);
4724   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
4725   PetscCall(DMPlexSetChart(dm, pStart, pEnd));
4726   for (p = pStart; p < pEnd; ++p) {
4727     PetscCall(DMPlexSetConeSize(dm, p, coneSize[p - pStart]));
4728     if (firstVertex < 0 && !coneSize[p - pStart]) { firstVertex = p - pStart; }
4729   }
4730   PetscCheck(firstVertex >= 0 || !numPoints[0], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Expected %" PetscInt_FMT " vertices but could not find any", numPoints[0]);
4731   PetscCall(DMSetUp(dm)); /* Allocate space for cones */
4732   for (p = pStart, off = 0; p < pEnd; off += coneSize[p - pStart], ++p) {
4733     PetscCall(DMPlexSetCone(dm, p, &cones[off]));
4734     PetscCall(DMPlexSetConeOrientation(dm, p, &coneOrientations[off]));
4735   }
4736   PetscCall(DMPlexSymmetrize(dm));
4737   PetscCall(DMPlexStratify(dm));
4738   /* Build coordinates */
4739   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4740   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4741   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dimEmbed));
4742   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numPoints[0]));
4743   for (v = firstVertex; v < firstVertex + numPoints[0]; ++v) {
4744     PetscCall(PetscSectionSetDof(coordSection, v, dimEmbed));
4745     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed));
4746   }
4747   PetscCall(PetscSectionSetUp(coordSection));
4748   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4749   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
4750   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4751   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4752   PetscCall(VecSetBlockSize(coordinates, dimEmbed));
4753   PetscCall(VecSetType(coordinates, VECSTANDARD));
4754   if (vertexCoords) {
4755     PetscCall(VecGetArray(coordinates, &coords));
4756     for (v = 0; v < numPoints[0]; ++v) {
4757       PetscInt off;
4758 
4759       PetscCall(PetscSectionGetOffset(coordSection, v + firstVertex, &off));
4760       for (d = 0; d < dimEmbed; ++d) { coords[off + d] = vertexCoords[v * dimEmbed + d]; }
4761     }
4762   }
4763   PetscCall(VecRestoreArray(coordinates, &coords));
4764   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4765   PetscCall(VecDestroy(&coordinates));
4766   PetscFunctionReturn(0);
4767 }
4768 
4769 /*@C
4770   DMPlexCreateCellVertexFromFile - Create a DMPlex mesh from a simple cell-vertex file.
4771 
4772 + comm        - The MPI communicator
4773 . filename    - Name of the .dat file
4774 - interpolate - Create faces and edges in the mesh
4775 
4776   Output Parameter:
4777 . dm  - The DM object representing the mesh
4778 
4779   Note: The format is the simplest possible:
4780 $ Ne
4781 $ v0 v1 ... vk
4782 $ Nv
4783 $ x y z marker
4784 
4785   Level: beginner
4786 
4787 .seealso: `DMPlexCreateFromFile()`, `DMPlexCreateMedFromFile()`, `DMPlexCreateGmsh()`, `DMPlexCreate()`
4788 @*/
4789 PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm) {
4790   DMLabel      marker;
4791   PetscViewer  viewer;
4792   Vec          coordinates;
4793   PetscSection coordSection;
4794   PetscScalar *coords;
4795   char         line[PETSC_MAX_PATH_LEN];
4796   PetscInt     dim = 3, cdim = 3, coordSize, v, c, d;
4797   PetscMPIInt  rank;
4798   int          snum, Nv, Nc, Ncn, Nl;
4799 
4800   PetscFunctionBegin;
4801   PetscCallMPI(MPI_Comm_rank(comm, &rank));
4802   PetscCall(PetscViewerCreate(comm, &viewer));
4803   PetscCall(PetscViewerSetType(viewer, PETSCVIEWERASCII));
4804   PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
4805   PetscCall(PetscViewerFileSetName(viewer, filename));
4806   if (rank == 0) {
4807     PetscCall(PetscViewerRead(viewer, line, 4, NULL, PETSC_STRING));
4808     snum = sscanf(line, "%d %d %d %d", &Nc, &Nv, &Ncn, &Nl);
4809     PetscCheck(snum == 4, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4810   } else {
4811     Nc = Nv = Ncn = Nl = 0;
4812   }
4813   PetscCall(DMCreate(comm, dm));
4814   PetscCall(DMSetType(*dm, DMPLEX));
4815   PetscCall(DMPlexSetChart(*dm, 0, Nc + Nv));
4816   PetscCall(DMSetDimension(*dm, dim));
4817   PetscCall(DMSetCoordinateDim(*dm, cdim));
4818   /* Read topology */
4819   if (rank == 0) {
4820     char     format[PETSC_MAX_PATH_LEN];
4821     PetscInt cone[8];
4822     int      vbuf[8], v;
4823 
4824     for (c = 0; c < Ncn; ++c) {
4825       format[c * 3 + 0] = '%';
4826       format[c * 3 + 1] = 'd';
4827       format[c * 3 + 2] = ' ';
4828     }
4829     format[Ncn * 3 - 1] = '\0';
4830     for (c = 0; c < Nc; ++c) PetscCall(DMPlexSetConeSize(*dm, c, Ncn));
4831     PetscCall(DMSetUp(*dm));
4832     for (c = 0; c < Nc; ++c) {
4833       PetscCall(PetscViewerRead(viewer, line, Ncn, NULL, PETSC_STRING));
4834       switch (Ncn) {
4835       case 2: snum = sscanf(line, format, &vbuf[0], &vbuf[1]); break;
4836       case 3: snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2]); break;
4837       case 4: snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3]); break;
4838       case 6: snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5]); break;
4839       case 8: snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]); break;
4840       default: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No cell shape with %d vertices", Ncn);
4841       }
4842       PetscCheck(snum == Ncn, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4843       for (v = 0; v < Ncn; ++v) cone[v] = vbuf[v] + Nc;
4844       /* Hexahedra are inverted */
4845       if (Ncn == 8) {
4846         PetscInt tmp = cone[1];
4847         cone[1]      = cone[3];
4848         cone[3]      = tmp;
4849       }
4850       PetscCall(DMPlexSetCone(*dm, c, cone));
4851     }
4852   }
4853   PetscCall(DMPlexSymmetrize(*dm));
4854   PetscCall(DMPlexStratify(*dm));
4855   /* Read coordinates */
4856   PetscCall(DMGetCoordinateSection(*dm, &coordSection));
4857   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4858   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
4859   PetscCall(PetscSectionSetChart(coordSection, Nc, Nc + Nv));
4860   for (v = Nc; v < Nc + Nv; ++v) {
4861     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
4862     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
4863   }
4864   PetscCall(PetscSectionSetUp(coordSection));
4865   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4866   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
4867   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4868   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4869   PetscCall(VecSetBlockSize(coordinates, cdim));
4870   PetscCall(VecSetType(coordinates, VECSTANDARD));
4871   PetscCall(VecGetArray(coordinates, &coords));
4872   if (rank == 0) {
4873     char   format[PETSC_MAX_PATH_LEN];
4874     double x[3];
4875     int    l, val[3];
4876 
4877     if (Nl) {
4878       for (l = 0; l < Nl; ++l) {
4879         format[l * 3 + 0] = '%';
4880         format[l * 3 + 1] = 'd';
4881         format[l * 3 + 2] = ' ';
4882       }
4883       format[Nl * 3 - 1] = '\0';
4884       PetscCall(DMCreateLabel(*dm, "marker"));
4885       PetscCall(DMGetLabel(*dm, "marker", &marker));
4886     }
4887     for (v = 0; v < Nv; ++v) {
4888       PetscCall(PetscViewerRead(viewer, line, 3 + Nl, NULL, PETSC_STRING));
4889       snum = sscanf(line, "%lg %lg %lg", &x[0], &x[1], &x[2]);
4890       PetscCheck(snum == 3, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4891       switch (Nl) {
4892       case 0: snum = 0; break;
4893       case 1: snum = sscanf(line, format, &val[0]); break;
4894       case 2: snum = sscanf(line, format, &val[0], &val[1]); break;
4895       case 3: snum = sscanf(line, format, &val[0], &val[1], &val[2]); break;
4896       default: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Request support for %d labels", Nl);
4897       }
4898       PetscCheck(snum == Nl, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4899       for (d = 0; d < cdim; ++d) coords[v * cdim + d] = x[d];
4900       for (l = 0; l < Nl; ++l) PetscCall(DMLabelSetValue(marker, v + Nc, val[l]));
4901     }
4902   }
4903   PetscCall(VecRestoreArray(coordinates, &coords));
4904   PetscCall(DMSetCoordinatesLocal(*dm, coordinates));
4905   PetscCall(VecDestroy(&coordinates));
4906   PetscCall(PetscViewerDestroy(&viewer));
4907   if (interpolate) {
4908     DM      idm;
4909     DMLabel bdlabel;
4910 
4911     PetscCall(DMPlexInterpolate(*dm, &idm));
4912     PetscCall(DMDestroy(dm));
4913     *dm = idm;
4914 
4915     if (!Nl) {
4916       PetscCall(DMCreateLabel(*dm, "marker"));
4917       PetscCall(DMGetLabel(*dm, "marker", &bdlabel));
4918       PetscCall(DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel));
4919       PetscCall(DMPlexLabelComplete(*dm, bdlabel));
4920     }
4921   }
4922   PetscFunctionReturn(0);
4923 }
4924 
4925 /*@C
4926   DMPlexCreateFromFile - This takes a filename and produces a DM
4927 
4928   Input Parameters:
4929 + comm - The communicator
4930 . filename - A file name
4931 . plexname - The object name of the resulting DM, also used for intra-datafile lookup by some formats
4932 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
4933 
4934   Output Parameter:
4935 . dm - The DM
4936 
4937   Options Database Keys:
4938 . -dm_plex_create_from_hdf5_xdmf - use the PETSC_VIEWER_HDF5_XDMF format for reading HDF5
4939 
4940   Use -dm_plex_create_ prefix to pass options to the internal PetscViewer, e.g.
4941 $ -dm_plex_create_viewer_hdf5_collective
4942 
4943   Notes:
4944   Using PETSCVIEWERHDF5 type with PETSC_VIEWER_HDF5_PETSC format, one can save multiple DMPlex
4945   meshes in a single HDF5 file. This in turn requires one to name the DMPlex object with PetscObjectSetName()
4946   before saving it with DMView() and before loading it with DMLoad() for identification of the mesh object.
4947   The input parameter name is thus used to name the DMPlex object when DMPlexCreateFromFile() internally
4948   calls DMLoad(). Currently, name is ignored for other viewer types and/or formats.
4949 
4950   Level: beginner
4951 
4952 .seealso: `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`, `PetscObjectSetName()`, `DMView()`, `DMLoad()`
4953 @*/
4954 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], const char plexname[], PetscBool interpolate, DM *dm) {
4955   const char  extGmsh[]      = ".msh";
4956   const char  extGmsh2[]     = ".msh2";
4957   const char  extGmsh4[]     = ".msh4";
4958   const char  extCGNS[]      = ".cgns";
4959   const char  extExodus[]    = ".exo";
4960   const char  extExodus_e[]  = ".e";
4961   const char  extGenesis[]   = ".gen";
4962   const char  extFluent[]    = ".cas";
4963   const char  extHDF5[]      = ".h5";
4964   const char  extMed[]       = ".med";
4965   const char  extPLY[]       = ".ply";
4966   const char  extEGADSLite[] = ".egadslite";
4967   const char  extEGADS[]     = ".egads";
4968   const char  extIGES[]      = ".igs";
4969   const char  extSTEP[]      = ".stp";
4970   const char  extCV[]        = ".dat";
4971   size_t      len;
4972   PetscBool   isGmsh, isGmsh2, isGmsh4, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isMed, isPLY, isEGADSLite, isEGADS, isIGES, isSTEP, isCV;
4973   PetscMPIInt rank;
4974 
4975   PetscFunctionBegin;
4976   PetscValidCharPointer(filename, 2);
4977   if (plexname) PetscValidCharPointer(plexname, 3);
4978   PetscValidPointer(dm, 5);
4979   PetscCall(DMInitializePackage());
4980   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromFile, 0, 0, 0, 0));
4981   PetscCallMPI(MPI_Comm_rank(comm, &rank));
4982   PetscCall(PetscStrlen(filename, &len));
4983   PetscCheck(len, comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
4984 
4985 #define CheckExtension(extension__, is_extension__) \
4986   do { \
4987     PetscAssert(sizeof(extension__), comm, PETSC_ERR_PLIB, "Zero-size extension: %s", extension__); \
4988     /* don't count the null-terminator at the end */ \
4989     const size_t ext_len = sizeof(extension__) - 1; \
4990     if (len < ext_len) { \
4991       is_extension__ = PETSC_FALSE; \
4992     } else { \
4993       PetscCall(PetscStrncmp(filename + len - ext_len, extension__, ext_len, &is_extension__)); \
4994     } \
4995   } while (0)
4996 
4997   CheckExtension(extGmsh, isGmsh);
4998   CheckExtension(extGmsh2, isGmsh2);
4999   CheckExtension(extGmsh4, isGmsh4);
5000   CheckExtension(extCGNS, isCGNS);
5001   CheckExtension(extExodus, isExodus);
5002   if (!isExodus) CheckExtension(extExodus_e, isExodus);
5003   CheckExtension(extGenesis, isGenesis);
5004   CheckExtension(extFluent, isFluent);
5005   CheckExtension(extHDF5, isHDF5);
5006   CheckExtension(extMed, isMed);
5007   CheckExtension(extPLY, isPLY);
5008   CheckExtension(extEGADSLite, isEGADSLite);
5009   CheckExtension(extEGADS, isEGADS);
5010   CheckExtension(extIGES, isIGES);
5011   CheckExtension(extSTEP, isSTEP);
5012   CheckExtension(extCV, isCV);
5013 
5014 #undef CheckExtension
5015 
5016   if (isGmsh || isGmsh2 || isGmsh4) {
5017     PetscCall(DMPlexCreateGmshFromFile(comm, filename, interpolate, dm));
5018   } else if (isCGNS) {
5019     PetscCall(DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm));
5020   } else if (isExodus || isGenesis) {
5021     PetscCall(DMPlexCreateExodusFromFile(comm, filename, interpolate, dm));
5022   } else if (isFluent) {
5023     PetscCall(DMPlexCreateFluentFromFile(comm, filename, interpolate, dm));
5024   } else if (isHDF5) {
5025     PetscBool   load_hdf5_xdmf = PETSC_FALSE;
5026     PetscViewer viewer;
5027 
5028     /* PETSC_VIEWER_HDF5_XDMF is used if the filename ends with .xdmf.h5, or if -dm_plex_create_from_hdf5_xdmf option is present */
5029     PetscCall(PetscStrncmp(&filename[PetscMax(0, len - 8)], ".xdmf", 5, &load_hdf5_xdmf));
5030     PetscCall(PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &load_hdf5_xdmf, NULL));
5031     PetscCall(PetscViewerCreate(comm, &viewer));
5032     PetscCall(PetscViewerSetType(viewer, PETSCVIEWERHDF5));
5033     PetscCall(PetscViewerSetOptionsPrefix(viewer, "dm_plex_create_"));
5034     PetscCall(PetscViewerSetFromOptions(viewer));
5035     PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
5036     PetscCall(PetscViewerFileSetName(viewer, filename));
5037 
5038     PetscCall(DMCreate(comm, dm));
5039     PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5040     PetscCall(DMSetType(*dm, DMPLEX));
5041     if (load_hdf5_xdmf) PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF));
5042     PetscCall(DMLoad(*dm, viewer));
5043     if (load_hdf5_xdmf) PetscCall(PetscViewerPopFormat(viewer));
5044     PetscCall(PetscViewerDestroy(&viewer));
5045 
5046     if (interpolate) {
5047       DM idm;
5048 
5049       PetscCall(DMPlexInterpolate(*dm, &idm));
5050       PetscCall(DMDestroy(dm));
5051       *dm = idm;
5052     }
5053   } else if (isMed) {
5054     PetscCall(DMPlexCreateMedFromFile(comm, filename, interpolate, dm));
5055   } else if (isPLY) {
5056     PetscCall(DMPlexCreatePLYFromFile(comm, filename, interpolate, dm));
5057   } else if (isEGADSLite || isEGADS || isIGES || isSTEP) {
5058     if (isEGADSLite) PetscCall(DMPlexCreateEGADSLiteFromFile(comm, filename, dm));
5059     else PetscCall(DMPlexCreateEGADSFromFile(comm, filename, dm));
5060     if (!interpolate) {
5061       DM udm;
5062 
5063       PetscCall(DMPlexUninterpolate(*dm, &udm));
5064       PetscCall(DMDestroy(dm));
5065       *dm = udm;
5066     }
5067   } else if (isCV) {
5068     PetscCall(DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm));
5069   } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
5070   PetscCall(PetscStrlen(plexname, &len));
5071   if (len) PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5072   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5073   PetscFunctionReturn(0);
5074 }
5075