xref: /petsc/src/dm/impls/plex/plexcreate.c (revision ff6450cf88c38d9373e3b4bf5fb1660da3ed7cfa)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h> /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>   /*I   "petscdmplex.h"   I*/
4 #include <petscsf.h>
5 #include <petscdmplextransform.h>
6 #include <petsc/private/kernels/blockmatmult.h>
7 #include <petsc/private/kernels/blockinvert.h>
8 
9 PetscLogEvent DMPLEX_CreateFromFile, DMPLEX_BuildFromCellList, DMPLEX_BuildCoordinatesFromCellList;
10 
11 /* External function declarations here */
12 static PetscErrorCode DMInitialize_Plex(DM dm);
13 
14 /* This copies internal things in the Plex structure that we generally want when making a new, related Plex */
15 PetscErrorCode DMPlexCopy_Internal(DM dmin, PetscBool copyPeriodicity, PetscBool copyOverlap, DM dmout) {
16   const PetscReal         *maxCell, *Lstart, *L;
17   PetscBool                dist;
18   DMPlexReorderDefaultFlag reorder;
19 
20   PetscFunctionBegin;
21   if (copyPeriodicity) {
22     PetscCall(DMGetPeriodicity(dmin, &maxCell, &Lstart, &L));
23     PetscCall(DMSetPeriodicity(dmout, maxCell, Lstart, L));
24   }
25   PetscCall(DMPlexDistributeGetDefault(dmin, &dist));
26   PetscCall(DMPlexDistributeSetDefault(dmout, dist));
27   PetscCall(DMPlexReorderGetDefault(dmin, &reorder));
28   PetscCall(DMPlexReorderSetDefault(dmout, reorder));
29   ((DM_Plex *)dmout->data)->useHashLocation = ((DM_Plex *)dmin->data)->useHashLocation;
30   if (copyOverlap) PetscCall(DMPlexSetOverlap_Plex(dmout, dmin, 0));
31   PetscFunctionReturn(0);
32 }
33 
34 /* Replace dm with the contents of ndm, and then destroy ndm
35    - Share the DM_Plex structure
36    - Share the coordinates
37    - Share the SF
38 */
39 PetscErrorCode DMPlexReplace_Internal(DM dm, DM *ndm) {
40   PetscSF          sf;
41   DM               dmNew = *ndm, coordDM, coarseDM;
42   Vec              coords;
43   const PetscReal *maxCell, *Lstart, *L;
44   PetscInt         dim, cdim;
45 
46   PetscFunctionBegin;
47   if (dm == dmNew) {
48     PetscCall(DMDestroy(ndm));
49     PetscFunctionReturn(0);
50   }
51   dm->setupcalled = dmNew->setupcalled;
52   PetscCall(DMGetDimension(dmNew, &dim));
53   PetscCall(DMSetDimension(dm, dim));
54   PetscCall(DMGetCoordinateDim(dmNew, &cdim));
55   PetscCall(DMSetCoordinateDim(dm, cdim));
56   PetscCall(DMGetPointSF(dmNew, &sf));
57   PetscCall(DMSetPointSF(dm, sf));
58   PetscCall(DMGetCoordinateDM(dmNew, &coordDM));
59   PetscCall(DMGetCoordinatesLocal(dmNew, &coords));
60   PetscCall(DMSetCoordinateDM(dm, coordDM));
61   PetscCall(DMSetCoordinatesLocal(dm, coords));
62   PetscCall(DMGetCellCoordinateDM(dmNew, &coordDM));
63   PetscCall(DMGetCellCoordinatesLocal(dmNew, &coords));
64   PetscCall(DMSetCellCoordinateDM(dm, coordDM));
65   PetscCall(DMSetCellCoordinatesLocal(dm, coords));
66   /* Do not want to create the coordinate field if it does not already exist, so do not call DMGetCoordinateField() */
67   PetscCall(DMFieldDestroy(&dm->coordinates[0].field));
68   dm->coordinates[0].field            = dmNew->coordinates[0].field;
69   ((DM_Plex *)dmNew->data)->coordFunc = ((DM_Plex *)dm->data)->coordFunc;
70   PetscCall(DMGetPeriodicity(dmNew, &maxCell, &Lstart, &L));
71   PetscCall(DMSetPeriodicity(dm, maxCell, Lstart, L));
72   PetscCall(DMDestroy_Plex(dm));
73   PetscCall(DMInitialize_Plex(dm));
74   dm->data = dmNew->data;
75   ((DM_Plex *)dmNew->data)->refct++;
76   PetscCall(DMDestroyLabelLinkList_Internal(dm));
77   PetscCall(DMCopyLabels(dmNew, dm, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL));
78   PetscCall(DMGetCoarseDM(dmNew, &coarseDM));
79   PetscCall(DMSetCoarseDM(dm, coarseDM));
80   PetscCall(DMDestroy(ndm));
81   PetscFunctionReturn(0);
82 }
83 
84 /* Swap dm with the contents of dmNew
85    - Swap the DM_Plex structure
86    - Swap the coordinates
87    - Swap the point PetscSF
88 */
89 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB) {
90   DM          coordDMA, coordDMB;
91   Vec         coordsA, coordsB;
92   PetscSF     sfA, sfB;
93   DMField     fieldTmp;
94   void       *tmp;
95   DMLabelLink listTmp;
96   DMLabel     depthTmp;
97   PetscInt    tmpI;
98 
99   PetscFunctionBegin;
100   if (dmA == dmB) PetscFunctionReturn(0);
101   PetscCall(DMGetPointSF(dmA, &sfA));
102   PetscCall(DMGetPointSF(dmB, &sfB));
103   PetscCall(PetscObjectReference((PetscObject)sfA));
104   PetscCall(DMSetPointSF(dmA, sfB));
105   PetscCall(DMSetPointSF(dmB, sfA));
106   PetscCall(PetscObjectDereference((PetscObject)sfA));
107 
108   PetscCall(DMGetCoordinateDM(dmA, &coordDMA));
109   PetscCall(DMGetCoordinateDM(dmB, &coordDMB));
110   PetscCall(PetscObjectReference((PetscObject)coordDMA));
111   PetscCall(DMSetCoordinateDM(dmA, coordDMB));
112   PetscCall(DMSetCoordinateDM(dmB, coordDMA));
113   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
114 
115   PetscCall(DMGetCoordinatesLocal(dmA, &coordsA));
116   PetscCall(DMGetCoordinatesLocal(dmB, &coordsB));
117   PetscCall(PetscObjectReference((PetscObject)coordsA));
118   PetscCall(DMSetCoordinatesLocal(dmA, coordsB));
119   PetscCall(DMSetCoordinatesLocal(dmB, coordsA));
120   PetscCall(PetscObjectDereference((PetscObject)coordsA));
121 
122   PetscCall(DMGetCellCoordinateDM(dmA, &coordDMA));
123   PetscCall(DMGetCellCoordinateDM(dmB, &coordDMB));
124   PetscCall(PetscObjectReference((PetscObject)coordDMA));
125   PetscCall(DMSetCellCoordinateDM(dmA, coordDMB));
126   PetscCall(DMSetCellCoordinateDM(dmB, coordDMA));
127   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
128 
129   PetscCall(DMGetCellCoordinatesLocal(dmA, &coordsA));
130   PetscCall(DMGetCellCoordinatesLocal(dmB, &coordsB));
131   PetscCall(PetscObjectReference((PetscObject)coordsA));
132   PetscCall(DMSetCellCoordinatesLocal(dmA, coordsB));
133   PetscCall(DMSetCellCoordinatesLocal(dmB, coordsA));
134   PetscCall(PetscObjectDereference((PetscObject)coordsA));
135 
136   fieldTmp                  = dmA->coordinates[0].field;
137   dmA->coordinates[0].field = dmB->coordinates[0].field;
138   dmB->coordinates[0].field = fieldTmp;
139   fieldTmp                  = dmA->coordinates[1].field;
140   dmA->coordinates[1].field = dmB->coordinates[1].field;
141   dmB->coordinates[1].field = fieldTmp;
142   tmp                       = dmA->data;
143   dmA->data                 = dmB->data;
144   dmB->data                 = tmp;
145   listTmp                   = dmA->labels;
146   dmA->labels               = dmB->labels;
147   dmB->labels               = listTmp;
148   depthTmp                  = dmA->depthLabel;
149   dmA->depthLabel           = dmB->depthLabel;
150   dmB->depthLabel           = depthTmp;
151   depthTmp                  = dmA->celltypeLabel;
152   dmA->celltypeLabel        = dmB->celltypeLabel;
153   dmB->celltypeLabel        = depthTmp;
154   tmpI                      = dmA->levelup;
155   dmA->levelup              = dmB->levelup;
156   dmB->levelup              = tmpI;
157   PetscFunctionReturn(0);
158 }
159 
160 static PetscErrorCode DMPlexInterpolateInPlace_Internal(DM dm) {
161   DM idm;
162 
163   PetscFunctionBegin;
164   PetscCall(DMPlexInterpolate(dm, &idm));
165   PetscCall(DMPlexCopyCoordinates(dm, idm));
166   PetscCall(DMPlexReplace_Internal(dm, &idm));
167   PetscFunctionReturn(0);
168 }
169 
170 /*@C
171   DMPlexCreateCoordinateSpace - Creates a finite element space for the coordinates
172 
173   Collective
174 
175   Input Parameters:
176 + DM        - The DM
177 . degree    - The degree of the finite element or PETSC_DECIDE
178 - coordFunc - An optional function to map new points from refinement to the surface
179 
180   Level: advanced
181 
182 .seealso: `PetscFECreateLagrange()`, `DMGetCoordinateDM()`
183 @*/
184 PetscErrorCode DMPlexCreateCoordinateSpace(DM dm, PetscInt degree, PetscPointFunc coordFunc) {
185   DM_Plex     *mesh = (DM_Plex *)dm->data;
186   DM           cdm;
187   PetscDS      cds;
188   PetscFE      fe;
189   PetscClassId id;
190 
191   PetscFunctionBegin;
192   PetscCall(DMGetCoordinateDM(dm, &cdm));
193   PetscCall(DMGetDS(cdm, &cds));
194   PetscCall(PetscDSGetDiscretization(cds, 0, (PetscObject *)&fe));
195   PetscCall(PetscObjectGetClassId((PetscObject)fe, &id));
196   if (id != PETSCFE_CLASSID) {
197     PetscBool simplex;
198     PetscInt  dim, dE, qorder;
199 
200     PetscCall(DMGetDimension(dm, &dim));
201     PetscCall(DMGetCoordinateDim(dm, &dE));
202     qorder = degree;
203     PetscObjectOptionsBegin((PetscObject)cdm);
204     PetscCall(PetscOptionsBoundedInt("-coord_dm_default_quadrature_order", "Quadrature order is one less than quadrature points per edge", "DMPlexCreateCoordinateSpace", qorder, &qorder, NULL, 0));
205     PetscOptionsEnd();
206     if (degree == PETSC_DECIDE) fe = NULL;
207     else {
208       PetscCall(DMPlexIsSimplex(dm, &simplex));
209       PetscCall(PetscFECreateLagrange(PETSC_COMM_SELF, dim, dE, simplex, degree, qorder, &fe));
210     }
211     PetscCall(DMProjectCoordinates(dm, fe));
212     PetscCall(PetscFEDestroy(&fe));
213   }
214   mesh->coordFunc = coordFunc;
215   PetscFunctionReturn(0);
216 }
217 
218 /*@
219   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
220 
221   Collective
222 
223   Input Parameters:
224 + comm - The communicator for the DM object
225 . dim - The spatial dimension
226 . simplex - Flag for simplicial cells, otherwise they are tensor product cells
227 . interpolate - Flag to create intermediate mesh pieces (edges, faces)
228 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
229 
230   Output Parameter:
231 . dm - The DM object
232 
233   Level: beginner
234 
235 .seealso: `DMSetType()`, `DMCreate()`
236 @*/
237 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscReal refinementLimit, DM *newdm) {
238   DM          dm;
239   PetscMPIInt rank;
240 
241   PetscFunctionBegin;
242   PetscCall(DMCreate(comm, &dm));
243   PetscCall(DMSetType(dm, DMPLEX));
244   PetscCall(DMSetDimension(dm, dim));
245   PetscCallMPI(MPI_Comm_rank(comm, &rank));
246   switch (dim) {
247   case 2:
248     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "triangular"));
249     else PetscCall(PetscObjectSetName((PetscObject)dm, "quadrilateral"));
250     break;
251   case 3:
252     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "tetrahedral"));
253     else PetscCall(PetscObjectSetName((PetscObject)dm, "hexahedral"));
254     break;
255   default: SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
256   }
257   if (rank) {
258     PetscInt numPoints[2] = {0, 0};
259     PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL));
260   } else {
261     switch (dim) {
262     case 2:
263       if (simplex) {
264         PetscInt    numPoints[2]        = {4, 2};
265         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
266         PetscInt    cones[6]            = {2, 3, 4, 5, 4, 3};
267         PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
268         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
269 
270         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
271       } else {
272         PetscInt    numPoints[2]        = {6, 2};
273         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
274         PetscInt    cones[8]            = {2, 3, 4, 5, 3, 6, 7, 4};
275         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
276         PetscScalar vertexCoords[12]    = {-1.0, -0.5, 0.0, -0.5, 0.0, 0.5, -1.0, 0.5, 1.0, -0.5, 1.0, 0.5};
277 
278         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
279       }
280       break;
281     case 3:
282       if (simplex) {
283         PetscInt    numPoints[2]        = {5, 2};
284         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
285         PetscInt    cones[8]            = {4, 3, 5, 2, 5, 3, 4, 6};
286         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
287         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0};
288 
289         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
290       } else {
291         PetscInt    numPoints[2]         = {12, 2};
292         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
293         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9, 5, 4, 10, 11, 7, 12, 13, 8};
294         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
295         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5, -1.0, 0.5, -0.5, 0.0, 0.5, -0.5, 0.0, -0.5, -0.5, -1.0, -0.5, 0.5, 0.0, -0.5, 0.5, 0.0, 0.5, 0.5, -1.0, 0.5, 0.5, 1.0, 0.5, -0.5, 1.0, -0.5, -0.5, 1.0, -0.5, 0.5, 1.0, 0.5, 0.5};
296 
297         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
298       }
299       break;
300     default: SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
301     }
302   }
303   *newdm = dm;
304   if (refinementLimit > 0.0) {
305     DM          rdm;
306     const char *name;
307 
308     PetscCall(DMPlexSetRefinementUniform(*newdm, PETSC_FALSE));
309     PetscCall(DMPlexSetRefinementLimit(*newdm, refinementLimit));
310     PetscCall(DMRefine(*newdm, comm, &rdm));
311     PetscCall(PetscObjectGetName((PetscObject)*newdm, &name));
312     PetscCall(PetscObjectSetName((PetscObject)rdm, name));
313     PetscCall(DMDestroy(newdm));
314     *newdm = rdm;
315   }
316   if (interpolate) {
317     DM idm;
318 
319     PetscCall(DMPlexInterpolate(*newdm, &idm));
320     PetscCall(DMDestroy(newdm));
321     *newdm = idm;
322   }
323   PetscFunctionReturn(0);
324 }
325 
326 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[]) {
327   const PetscInt numVertices    = 2;
328   PetscInt       markerRight    = 1;
329   PetscInt       markerLeft     = 1;
330   PetscBool      markerSeparate = PETSC_FALSE;
331   Vec            coordinates;
332   PetscSection   coordSection;
333   PetscScalar   *coords;
334   PetscInt       coordSize;
335   PetscMPIInt    rank;
336   PetscInt       cdim = 1, v;
337 
338   PetscFunctionBegin;
339   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
340   if (markerSeparate) {
341     markerRight = 2;
342     markerLeft  = 1;
343   }
344   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
345   if (rank == 0) {
346     PetscCall(DMPlexSetChart(dm, 0, numVertices));
347     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
348     PetscCall(DMSetLabelValue(dm, "marker", 0, markerLeft));
349     PetscCall(DMSetLabelValue(dm, "marker", 1, markerRight));
350   }
351   PetscCall(DMPlexSymmetrize(dm));
352   PetscCall(DMPlexStratify(dm));
353   /* Build coordinates */
354   PetscCall(DMSetCoordinateDim(dm, cdim));
355   PetscCall(DMGetCoordinateSection(dm, &coordSection));
356   PetscCall(PetscSectionSetNumFields(coordSection, 1));
357   PetscCall(PetscSectionSetChart(coordSection, 0, numVertices));
358   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
359   for (v = 0; v < numVertices; ++v) {
360     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
361     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
362   }
363   PetscCall(PetscSectionSetUp(coordSection));
364   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
365   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
366   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
367   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
368   PetscCall(VecSetBlockSize(coordinates, cdim));
369   PetscCall(VecSetType(coordinates, VECSTANDARD));
370   PetscCall(VecGetArray(coordinates, &coords));
371   coords[0] = lower[0];
372   coords[1] = upper[0];
373   PetscCall(VecRestoreArray(coordinates, &coords));
374   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
375   PetscCall(VecDestroy(&coordinates));
376   PetscFunctionReturn(0);
377 }
378 
379 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[]) {
380   const PetscInt numVertices    = (edges[0] + 1) * (edges[1] + 1);
381   const PetscInt numEdges       = edges[0] * (edges[1] + 1) + (edges[0] + 1) * edges[1];
382   PetscInt       markerTop      = 1;
383   PetscInt       markerBottom   = 1;
384   PetscInt       markerRight    = 1;
385   PetscInt       markerLeft     = 1;
386   PetscBool      markerSeparate = PETSC_FALSE;
387   Vec            coordinates;
388   PetscSection   coordSection;
389   PetscScalar   *coords;
390   PetscInt       coordSize;
391   PetscMPIInt    rank;
392   PetscInt       v, vx, vy;
393 
394   PetscFunctionBegin;
395   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
396   if (markerSeparate) {
397     markerTop    = 3;
398     markerBottom = 1;
399     markerRight  = 2;
400     markerLeft   = 4;
401   }
402   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
403   if (rank == 0) {
404     PetscInt e, ex, ey;
405 
406     PetscCall(DMPlexSetChart(dm, 0, numEdges + numVertices));
407     for (e = 0; e < numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
408     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
409     for (vx = 0; vx <= edges[0]; vx++) {
410       for (ey = 0; ey < edges[1]; ey++) {
411         PetscInt edge   = vx * edges[1] + ey + edges[0] * (edges[1] + 1);
412         PetscInt vertex = ey * (edges[0] + 1) + vx + numEdges;
413         PetscInt cone[2];
414 
415         cone[0] = vertex;
416         cone[1] = vertex + edges[0] + 1;
417         PetscCall(DMPlexSetCone(dm, edge, cone));
418         if (vx == edges[0]) {
419           PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
420           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
421           if (ey == edges[1] - 1) {
422             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
423             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerRight));
424           }
425         } else if (vx == 0) {
426           PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
427           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
428           if (ey == edges[1] - 1) {
429             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
430             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft));
431           }
432         }
433       }
434     }
435     for (vy = 0; vy <= edges[1]; vy++) {
436       for (ex = 0; ex < edges[0]; ex++) {
437         PetscInt edge   = vy * edges[0] + ex;
438         PetscInt vertex = vy * (edges[0] + 1) + ex + numEdges;
439         PetscInt cone[2];
440 
441         cone[0] = vertex;
442         cone[1] = vertex + 1;
443         PetscCall(DMPlexSetCone(dm, edge, cone));
444         if (vy == edges[1]) {
445           PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
446           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
447           if (ex == edges[0] - 1) {
448             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
449             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerTop));
450           }
451         } else if (vy == 0) {
452           PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
453           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
454           if (ex == edges[0] - 1) {
455             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
456             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom));
457           }
458         }
459       }
460     }
461   }
462   PetscCall(DMPlexSymmetrize(dm));
463   PetscCall(DMPlexStratify(dm));
464   /* Build coordinates */
465   PetscCall(DMSetCoordinateDim(dm, 2));
466   PetscCall(DMGetCoordinateSection(dm, &coordSection));
467   PetscCall(PetscSectionSetNumFields(coordSection, 1));
468   PetscCall(PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices));
469   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 2));
470   for (v = numEdges; v < numEdges + numVertices; ++v) {
471     PetscCall(PetscSectionSetDof(coordSection, v, 2));
472     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 2));
473   }
474   PetscCall(PetscSectionSetUp(coordSection));
475   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
476   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
477   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
478   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
479   PetscCall(VecSetBlockSize(coordinates, 2));
480   PetscCall(VecSetType(coordinates, VECSTANDARD));
481   PetscCall(VecGetArray(coordinates, &coords));
482   for (vy = 0; vy <= edges[1]; ++vy) {
483     for (vx = 0; vx <= edges[0]; ++vx) {
484       coords[(vy * (edges[0] + 1) + vx) * 2 + 0] = lower[0] + ((upper[0] - lower[0]) / edges[0]) * vx;
485       coords[(vy * (edges[0] + 1) + vx) * 2 + 1] = lower[1] + ((upper[1] - lower[1]) / edges[1]) * vy;
486     }
487   }
488   PetscCall(VecRestoreArray(coordinates, &coords));
489   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
490   PetscCall(VecDestroy(&coordinates));
491   PetscFunctionReturn(0);
492 }
493 
494 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[]) {
495   PetscInt     vertices[3], numVertices;
496   PetscInt     numFaces       = 2 * faces[0] * faces[1] + 2 * faces[1] * faces[2] + 2 * faces[0] * faces[2];
497   PetscInt     markerTop      = 1;
498   PetscInt     markerBottom   = 1;
499   PetscInt     markerFront    = 1;
500   PetscInt     markerBack     = 1;
501   PetscInt     markerRight    = 1;
502   PetscInt     markerLeft     = 1;
503   PetscBool    markerSeparate = PETSC_FALSE;
504   Vec          coordinates;
505   PetscSection coordSection;
506   PetscScalar *coords;
507   PetscInt     coordSize;
508   PetscMPIInt  rank;
509   PetscInt     v, vx, vy, vz;
510   PetscInt     voffset, iface = 0, cone[4];
511 
512   PetscFunctionBegin;
513   PetscCheck(faces[0] >= 1 && faces[1] >= 1 && faces[2] >= 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
514   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
515   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
516   if (markerSeparate) {
517     markerBottom = 1;
518     markerTop    = 2;
519     markerFront  = 3;
520     markerBack   = 4;
521     markerRight  = 5;
522     markerLeft   = 6;
523   }
524   vertices[0] = faces[0] + 1;
525   vertices[1] = faces[1] + 1;
526   vertices[2] = faces[2] + 1;
527   numVertices = vertices[0] * vertices[1] * vertices[2];
528   if (rank == 0) {
529     PetscInt f;
530 
531     PetscCall(DMPlexSetChart(dm, 0, numFaces + numVertices));
532     for (f = 0; f < numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
533     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
534 
535     /* Side 0 (Top) */
536     for (vy = 0; vy < faces[1]; vy++) {
537       for (vx = 0; vx < faces[0]; vx++) {
538         voffset = numFaces + vertices[0] * vertices[1] * (vertices[2] - 1) + vy * vertices[0] + vx;
539         cone[0] = voffset;
540         cone[1] = voffset + 1;
541         cone[2] = voffset + vertices[0] + 1;
542         cone[3] = voffset + vertices[0];
543         PetscCall(DMPlexSetCone(dm, iface, cone));
544         PetscCall(DMSetLabelValue(dm, "marker", iface, markerTop));
545         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerTop));
546         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerTop));
547         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerTop));
548         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerTop));
549         iface++;
550       }
551     }
552 
553     /* Side 1 (Bottom) */
554     for (vy = 0; vy < faces[1]; vy++) {
555       for (vx = 0; vx < faces[0]; vx++) {
556         voffset = numFaces + vy * (faces[0] + 1) + vx;
557         cone[0] = voffset + 1;
558         cone[1] = voffset;
559         cone[2] = voffset + vertices[0];
560         cone[3] = voffset + vertices[0] + 1;
561         PetscCall(DMPlexSetCone(dm, iface, cone));
562         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBottom));
563         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBottom));
564         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBottom));
565         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerBottom));
566         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerBottom));
567         iface++;
568       }
569     }
570 
571     /* Side 2 (Front) */
572     for (vz = 0; vz < faces[2]; vz++) {
573       for (vx = 0; vx < faces[0]; vx++) {
574         voffset = numFaces + vz * vertices[0] * vertices[1] + vx;
575         cone[0] = voffset;
576         cone[1] = voffset + 1;
577         cone[2] = voffset + vertices[0] * vertices[1] + 1;
578         cone[3] = voffset + vertices[0] * vertices[1];
579         PetscCall(DMPlexSetCone(dm, iface, cone));
580         PetscCall(DMSetLabelValue(dm, "marker", iface, markerFront));
581         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerFront));
582         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerFront));
583         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerFront));
584         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerFront));
585         iface++;
586       }
587     }
588 
589     /* Side 3 (Back) */
590     for (vz = 0; vz < faces[2]; vz++) {
591       for (vx = 0; vx < faces[0]; vx++) {
592         voffset = numFaces + vz * vertices[0] * vertices[1] + vertices[0] * (vertices[1] - 1) + vx;
593         cone[0] = voffset + vertices[0] * vertices[1];
594         cone[1] = voffset + vertices[0] * vertices[1] + 1;
595         cone[2] = voffset + 1;
596         cone[3] = voffset;
597         PetscCall(DMPlexSetCone(dm, iface, cone));
598         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBack));
599         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBack));
600         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBack));
601         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerBack));
602         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerBack));
603         iface++;
604       }
605     }
606 
607     /* Side 4 (Left) */
608     for (vz = 0; vz < faces[2]; vz++) {
609       for (vy = 0; vy < faces[1]; vy++) {
610         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0];
611         cone[0] = voffset;
612         cone[1] = voffset + vertices[0] * vertices[1];
613         cone[2] = voffset + vertices[0] * vertices[1] + vertices[0];
614         cone[3] = voffset + vertices[0];
615         PetscCall(DMPlexSetCone(dm, iface, cone));
616         PetscCall(DMSetLabelValue(dm, "marker", iface, markerLeft));
617         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerLeft));
618         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerLeft));
619         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[1] + 0, markerLeft));
620         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerLeft));
621         iface++;
622       }
623     }
624 
625     /* Side 5 (Right) */
626     for (vz = 0; vz < faces[2]; vz++) {
627       for (vy = 0; vy < faces[1]; vy++) {
628         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0] + faces[0];
629         cone[0] = voffset + vertices[0] * vertices[1];
630         cone[1] = voffset;
631         cone[2] = voffset + vertices[0];
632         cone[3] = voffset + vertices[0] * vertices[1] + vertices[0];
633         PetscCall(DMPlexSetCone(dm, iface, cone));
634         PetscCall(DMSetLabelValue(dm, "marker", iface, markerRight));
635         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerRight));
636         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerRight));
637         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerRight));
638         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerRight));
639         iface++;
640       }
641     }
642   }
643   PetscCall(DMPlexSymmetrize(dm));
644   PetscCall(DMPlexStratify(dm));
645   /* Build coordinates */
646   PetscCall(DMSetCoordinateDim(dm, 3));
647   PetscCall(DMGetCoordinateSection(dm, &coordSection));
648   PetscCall(PetscSectionSetNumFields(coordSection, 1));
649   PetscCall(PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices));
650   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 3));
651   for (v = numFaces; v < numFaces + numVertices; ++v) {
652     PetscCall(PetscSectionSetDof(coordSection, v, 3));
653     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 3));
654   }
655   PetscCall(PetscSectionSetUp(coordSection));
656   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
657   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
658   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
659   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
660   PetscCall(VecSetBlockSize(coordinates, 3));
661   PetscCall(VecSetType(coordinates, VECSTANDARD));
662   PetscCall(VecGetArray(coordinates, &coords));
663   for (vz = 0; vz <= faces[2]; ++vz) {
664     for (vy = 0; vy <= faces[1]; ++vy) {
665       for (vx = 0; vx <= faces[0]; ++vx) {
666         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 0] = lower[0] + ((upper[0] - lower[0]) / faces[0]) * vx;
667         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 1] = lower[1] + ((upper[1] - lower[1]) / faces[1]) * vy;
668         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 2] = lower[2] + ((upper[2] - lower[2]) / faces[2]) * vz;
669       }
670     }
671   }
672   PetscCall(VecRestoreArray(coordinates, &coords));
673   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
674   PetscCall(VecDestroy(&coordinates));
675   PetscFunctionReturn(0);
676 }
677 
678 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate) {
679   PetscFunctionBegin;
680   PetscValidLogicalCollectiveInt(dm, dim, 2);
681   PetscCall(DMSetDimension(dm, dim - 1));
682   PetscCall(DMSetCoordinateDim(dm, dim));
683   switch (dim) {
684   case 1: PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(dm, lower, upper, faces)); break;
685   case 2: PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(dm, lower, upper, faces)); break;
686   case 3: PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(dm, lower, upper, faces)); break;
687   default: SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Dimension not supported: %" PetscInt_FMT, dim);
688   }
689   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
690   PetscFunctionReturn(0);
691 }
692 
693 /*@C
694   DMPlexCreateBoxSurfaceMesh - Creates a mesh on the surface of the tensor product of unit intervals (box) using tensor cells (hexahedra).
695 
696   Collective
697 
698   Input Parameters:
699 + comm        - The communicator for the DM object
700 . dim         - The spatial dimension of the box, so the resulting mesh is has dimension dim-1
701 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
702 . lower       - The lower left corner, or NULL for (0, 0, 0)
703 . upper       - The upper right corner, or NULL for (1, 1, 1)
704 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
705 
706   Output Parameter:
707 . dm  - The DM object
708 
709   Level: beginner
710 
711 .seealso: `DMSetFromOptions()`, `DMPlexCreateBoxMesh()`, `DMPlexCreateFromFile()`, `DMSetType()`, `DMCreate()`
712 @*/
713 PetscErrorCode DMPlexCreateBoxSurfaceMesh(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate, DM *dm) {
714   PetscInt  fac[3] = {1, 1, 1};
715   PetscReal low[3] = {0, 0, 0};
716   PetscReal upp[3] = {1, 1, 1};
717 
718   PetscFunctionBegin;
719   PetscCall(DMCreate(comm, dm));
720   PetscCall(DMSetType(*dm, DMPLEX));
721   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(*dm, dim, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, interpolate));
722   PetscFunctionReturn(0);
723 }
724 
725 static PetscErrorCode DMPlexCreateLineMesh_Internal(DM dm, PetscInt segments, PetscReal lower, PetscReal upper, DMBoundaryType bd) {
726   PetscInt     i, fStart, fEnd, numCells = 0, numVerts = 0;
727   PetscInt     numPoints[2], *coneSize, *cones, *coneOrientations;
728   PetscScalar *vertexCoords;
729   PetscReal    L, maxCell;
730   PetscBool    markerSeparate = PETSC_FALSE;
731   PetscInt     markerLeft = 1, faceMarkerLeft = 1;
732   PetscInt     markerRight = 1, faceMarkerRight = 2;
733   PetscBool    wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
734   PetscMPIInt  rank;
735 
736   PetscFunctionBegin;
737   PetscValidPointer(dm, 1);
738 
739   PetscCall(DMSetDimension(dm, 1));
740   PetscCall(DMCreateLabel(dm, "marker"));
741   PetscCall(DMCreateLabel(dm, "Face Sets"));
742 
743   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
744   if (rank == 0) numCells = segments;
745   if (rank == 0) numVerts = segments + (wrap ? 0 : 1);
746 
747   numPoints[0] = numVerts;
748   numPoints[1] = numCells;
749   PetscCall(PetscMalloc4(numCells + numVerts, &coneSize, numCells * 2, &cones, numCells + numVerts, &coneOrientations, numVerts, &vertexCoords));
750   PetscCall(PetscArrayzero(coneOrientations, numCells + numVerts));
751   for (i = 0; i < numCells; ++i) coneSize[i] = 2;
752   for (i = 0; i < numVerts; ++i) coneSize[numCells + i] = 0;
753   for (i = 0; i < numCells; ++i) {
754     cones[2 * i]     = numCells + i % numVerts;
755     cones[2 * i + 1] = numCells + (i + 1) % numVerts;
756   }
757   for (i = 0; i < numVerts; ++i) vertexCoords[i] = lower + (upper - lower) * ((PetscReal)i / (PetscReal)numCells);
758   PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
759   PetscCall(PetscFree4(coneSize, cones, coneOrientations, vertexCoords));
760 
761   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
762   if (markerSeparate) {
763     markerLeft  = faceMarkerLeft;
764     markerRight = faceMarkerRight;
765   }
766   if (!wrap && rank == 0) {
767     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
768     PetscCall(DMSetLabelValue(dm, "marker", fStart, markerLeft));
769     PetscCall(DMSetLabelValue(dm, "marker", fEnd - 1, markerRight));
770     PetscCall(DMSetLabelValue(dm, "Face Sets", fStart, faceMarkerLeft));
771     PetscCall(DMSetLabelValue(dm, "Face Sets", fEnd - 1, faceMarkerRight));
772   }
773   if (wrap) {
774     L       = upper - lower;
775     maxCell = (PetscReal)1.1 * (L / (PetscReal)PetscMax(1, segments));
776     PetscCall(DMSetPeriodicity(dm, &maxCell, &lower, &L));
777   }
778   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
779   PetscFunctionReturn(0);
780 }
781 
782 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate) {
783   DM      boundary, vol;
784   DMLabel bdlabel;
785 
786   PetscFunctionBegin;
787   PetscValidPointer(dm, 1);
788   for (PetscInt i = 0; i < dim; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
789   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &boundary));
790   PetscCall(DMSetType(boundary, DMPLEX));
791   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(boundary, dim, faces, lower, upper, PETSC_FALSE));
792   PetscCall(DMPlexGenerate(boundary, NULL, interpolate, &vol));
793   PetscCall(DMGetLabel(vol, "marker", &bdlabel));
794   if (bdlabel) PetscCall(DMPlexLabelComplete(vol, bdlabel));
795   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_FALSE, vol));
796   PetscCall(DMPlexReplace_Internal(dm, &vol));
797   PetscCall(DMDestroy(&boundary));
798   PetscFunctionReturn(0);
799 }
800 
801 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ) {
802   DMLabel     cutLabel  = NULL;
803   PetscInt    markerTop = 1, faceMarkerTop = 1;
804   PetscInt    markerBottom = 1, faceMarkerBottom = 1;
805   PetscInt    markerFront = 1, faceMarkerFront = 1;
806   PetscInt    markerBack = 1, faceMarkerBack = 1;
807   PetscInt    markerRight = 1, faceMarkerRight = 1;
808   PetscInt    markerLeft = 1, faceMarkerLeft = 1;
809   PetscInt    dim;
810   PetscBool   markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
811   PetscMPIInt rank;
812 
813   PetscFunctionBegin;
814   PetscCall(DMGetDimension(dm, &dim));
815   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
816   PetscCall(DMCreateLabel(dm, "marker"));
817   PetscCall(DMCreateLabel(dm, "Face Sets"));
818   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
819   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST || bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST || bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
820     if (cutMarker) {
821       PetscCall(DMCreateLabel(dm, "periodic_cut"));
822       PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
823     }
824   }
825   switch (dim) {
826   case 2:
827     faceMarkerTop    = 3;
828     faceMarkerBottom = 1;
829     faceMarkerRight  = 2;
830     faceMarkerLeft   = 4;
831     break;
832   case 3:
833     faceMarkerBottom = 1;
834     faceMarkerTop    = 2;
835     faceMarkerFront  = 3;
836     faceMarkerBack   = 4;
837     faceMarkerRight  = 5;
838     faceMarkerLeft   = 6;
839     break;
840   default: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
841   }
842   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
843   if (markerSeparate) {
844     markerBottom = faceMarkerBottom;
845     markerTop    = faceMarkerTop;
846     markerFront  = faceMarkerFront;
847     markerBack   = faceMarkerBack;
848     markerRight  = faceMarkerRight;
849     markerLeft   = faceMarkerLeft;
850   }
851   {
852     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
853     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
854     const PetscInt numZEdges    = rank == 0 ? edges[2] : 0;
855     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0] + 1) : 0;
856     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1] + 1) : 0;
857     const PetscInt numZVertices = rank == 0 ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2] + 1) : 0;
858     const PetscInt numCells     = numXEdges * numYEdges * numZEdges;
859     const PetscInt numXFaces    = numYEdges * numZEdges;
860     const PetscInt numYFaces    = numXEdges * numZEdges;
861     const PetscInt numZFaces    = numXEdges * numYEdges;
862     const PetscInt numTotXFaces = numXVertices * numXFaces;
863     const PetscInt numTotYFaces = numYVertices * numYFaces;
864     const PetscInt numTotZFaces = numZVertices * numZFaces;
865     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
866     const PetscInt numTotXEdges = numXEdges * numYVertices * numZVertices;
867     const PetscInt numTotYEdges = numYEdges * numXVertices * numZVertices;
868     const PetscInt numTotZEdges = numZEdges * numXVertices * numYVertices;
869     const PetscInt numVertices  = numXVertices * numYVertices * numZVertices;
870     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
871     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
872     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
873     const PetscInt firstYFace   = firstXFace + numTotXFaces;
874     const PetscInt firstZFace   = firstYFace + numTotYFaces;
875     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
876     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
877     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
878     Vec            coordinates;
879     PetscSection   coordSection;
880     PetscScalar   *coords;
881     PetscInt       coordSize;
882     PetscInt       v, vx, vy, vz;
883     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
884 
885     PetscCall(DMPlexSetChart(dm, 0, numCells + numFaces + numEdges + numVertices));
886     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
887     for (f = firstXFace; f < firstXFace + numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
888     for (e = firstXEdge; e < firstXEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
889     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
890     /* Build cells */
891     for (fz = 0; fz < numZEdges; ++fz) {
892       for (fy = 0; fy < numYEdges; ++fy) {
893         for (fx = 0; fx < numXEdges; ++fx) {
894           PetscInt cell    = (fz * numYEdges + fy) * numXEdges + fx;
895           PetscInt faceB   = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
896           PetscInt faceT   = firstZFace + (fy * numXEdges + fx) * numZVertices + ((fz + 1) % numZVertices);
897           PetscInt faceF   = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
898           PetscInt faceK   = firstYFace + (fz * numXEdges + fx) * numYVertices + ((fy + 1) % numYVertices);
899           PetscInt faceL   = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
900           PetscInt faceR   = firstXFace + (fz * numYEdges + fy) * numXVertices + ((fx + 1) % numXVertices);
901           /* B,  T,  F,  K,  R,  L */
902           PetscInt ornt[6] = {-2, 0, 0, -3, 0, -2}; /* ??? */
903           PetscInt cone[6];
904 
905           /* no boundary twisting in 3D */
906           cone[0] = faceB;
907           cone[1] = faceT;
908           cone[2] = faceF;
909           cone[3] = faceK;
910           cone[4] = faceR;
911           cone[5] = faceL;
912           PetscCall(DMPlexSetCone(dm, cell, cone));
913           PetscCall(DMPlexSetConeOrientation(dm, cell, ornt));
914           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
915           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
916           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
917         }
918       }
919     }
920     /* Build x faces */
921     for (fz = 0; fz < numZEdges; ++fz) {
922       for (fy = 0; fy < numYEdges; ++fy) {
923         for (fx = 0; fx < numXVertices; ++fx) {
924           PetscInt face    = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
925           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
926           PetscInt edgeR   = firstZEdge + (((fy + 1) % numYVertices) * numXVertices + fx) * numZEdges + fz;
927           PetscInt edgeB   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
928           PetscInt edgeT   = firstYEdge + (((fz + 1) % numZVertices) * numXVertices + fx) * numYEdges + fy;
929           PetscInt ornt[4] = {0, 0, -1, -1};
930           PetscInt cone[4];
931 
932           if (dim == 3) {
933             /* markers */
934             if (bdX != DM_BOUNDARY_PERIODIC) {
935               if (fx == numXVertices - 1) {
936                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight));
937                 PetscCall(DMSetLabelValue(dm, "marker", face, markerRight));
938               } else if (fx == 0) {
939                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft));
940                 PetscCall(DMSetLabelValue(dm, "marker", face, markerLeft));
941               }
942             }
943           }
944           cone[0] = edgeB;
945           cone[1] = edgeR;
946           cone[2] = edgeT;
947           cone[3] = edgeL;
948           PetscCall(DMPlexSetCone(dm, face, cone));
949           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
950         }
951       }
952     }
953     /* Build y faces */
954     for (fz = 0; fz < numZEdges; ++fz) {
955       for (fx = 0; fx < numXEdges; ++fx) {
956         for (fy = 0; fy < numYVertices; ++fy) {
957           PetscInt face    = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
958           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
959           PetscInt edgeR   = firstZEdge + (fy * numXVertices + ((fx + 1) % numXVertices)) * numZEdges + fz;
960           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
961           PetscInt edgeT   = firstXEdge + (((fz + 1) % numZVertices) * numYVertices + fy) * numXEdges + fx;
962           PetscInt ornt[4] = {0, 0, -1, -1};
963           PetscInt cone[4];
964 
965           if (dim == 3) {
966             /* markers */
967             if (bdY != DM_BOUNDARY_PERIODIC) {
968               if (fy == numYVertices - 1) {
969                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack));
970                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBack));
971               } else if (fy == 0) {
972                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront));
973                 PetscCall(DMSetLabelValue(dm, "marker", face, markerFront));
974               }
975             }
976           }
977           cone[0] = edgeB;
978           cone[1] = edgeR;
979           cone[2] = edgeT;
980           cone[3] = edgeL;
981           PetscCall(DMPlexSetCone(dm, face, cone));
982           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
983         }
984       }
985     }
986     /* Build z faces */
987     for (fy = 0; fy < numYEdges; ++fy) {
988       for (fx = 0; fx < numXEdges; ++fx) {
989         for (fz = 0; fz < numZVertices; fz++) {
990           PetscInt face    = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
991           PetscInt edgeL   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
992           PetscInt edgeR   = firstYEdge + (fz * numXVertices + ((fx + 1) % numXVertices)) * numYEdges + fy;
993           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
994           PetscInt edgeT   = firstXEdge + (fz * numYVertices + ((fy + 1) % numYVertices)) * numXEdges + fx;
995           PetscInt ornt[4] = {0, 0, -1, -1};
996           PetscInt cone[4];
997 
998           if (dim == 2) {
999             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges - 1) {
1000               edgeR += numYEdges - 1 - 2 * fy;
1001               ornt[1] = -1;
1002             }
1003             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges - 1) {
1004               edgeT += numXEdges - 1 - 2 * fx;
1005               ornt[2] = 0;
1006             }
1007             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1008             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1009           } else {
1010             /* markers */
1011             if (bdZ != DM_BOUNDARY_PERIODIC) {
1012               if (fz == numZVertices - 1) {
1013                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop));
1014                 PetscCall(DMSetLabelValue(dm, "marker", face, markerTop));
1015               } else if (fz == 0) {
1016                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom));
1017                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBottom));
1018               }
1019             }
1020           }
1021           cone[0] = edgeB;
1022           cone[1] = edgeR;
1023           cone[2] = edgeT;
1024           cone[3] = edgeL;
1025           PetscCall(DMPlexSetCone(dm, face, cone));
1026           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1027         }
1028       }
1029     }
1030     /* Build Z edges*/
1031     for (vy = 0; vy < numYVertices; vy++) {
1032       for (vx = 0; vx < numXVertices; vx++) {
1033         for (ez = 0; ez < numZEdges; ez++) {
1034           const PetscInt edge    = firstZEdge + (vy * numXVertices + vx) * numZEdges + ez;
1035           const PetscInt vertexB = firstVertex + (ez * numYVertices + vy) * numXVertices + vx;
1036           const PetscInt vertexT = firstVertex + (((ez + 1) % numZVertices) * numYVertices + vy) * numXVertices + vx;
1037           PetscInt       cone[2];
1038 
1039           cone[0] = vertexB;
1040           cone[1] = vertexT;
1041           PetscCall(DMPlexSetCone(dm, edge, cone));
1042           if (dim == 3) {
1043             if (bdX != DM_BOUNDARY_PERIODIC) {
1044               if (vx == numXVertices - 1) {
1045                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1046                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1047                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1048               } else if (vx == 0) {
1049                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1050                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1051                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1052               }
1053             }
1054             if (bdY != DM_BOUNDARY_PERIODIC) {
1055               if (vy == numYVertices - 1) {
1056                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1057                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1058                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1059               } else if (vy == 0) {
1060                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1061                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1062                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1063               }
1064             }
1065           }
1066         }
1067       }
1068     }
1069     /* Build Y edges*/
1070     for (vz = 0; vz < numZVertices; vz++) {
1071       for (vx = 0; vx < numXVertices; vx++) {
1072         for (ey = 0; ey < numYEdges; ey++) {
1073           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges - 1) ? (numXVertices - vx - 1) : (vz * numYVertices + ((ey + 1) % numYVertices)) * numXVertices + vx;
1074           const PetscInt edge    = firstYEdge + (vz * numXVertices + vx) * numYEdges + ey;
1075           const PetscInt vertexF = firstVertex + (vz * numYVertices + ey) * numXVertices + vx;
1076           const PetscInt vertexK = firstVertex + nextv;
1077           PetscInt       cone[2];
1078 
1079           cone[0] = vertexF;
1080           cone[1] = vertexK;
1081           PetscCall(DMPlexSetCone(dm, edge, cone));
1082           if (dim == 2) {
1083             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1084               if (vx == numXVertices - 1) {
1085                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight));
1086                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1087                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1088                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1089               } else if (vx == 0) {
1090                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft));
1091                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1092                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1093                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1094               }
1095             } else {
1096               if (vx == 0 && cutLabel) {
1097                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1098                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1099                 if (ey == numYEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1100               }
1101             }
1102           } else {
1103             if (bdX != DM_BOUNDARY_PERIODIC) {
1104               if (vx == numXVertices - 1) {
1105                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1106                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1107                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1108               } else if (vx == 0) {
1109                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1110                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1111                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1112               }
1113             }
1114             if (bdZ != DM_BOUNDARY_PERIODIC) {
1115               if (vz == numZVertices - 1) {
1116                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1117                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1118                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1119               } else if (vz == 0) {
1120                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1121                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1122                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1123               }
1124             }
1125           }
1126         }
1127       }
1128     }
1129     /* Build X edges*/
1130     for (vz = 0; vz < numZVertices; vz++) {
1131       for (vy = 0; vy < numYVertices; vy++) {
1132         for (ex = 0; ex < numXEdges; ex++) {
1133           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges - 1) ? (numYVertices - vy - 1) * numXVertices : (vz * numYVertices + vy) * numXVertices + (ex + 1) % numXVertices;
1134           const PetscInt edge    = firstXEdge + (vz * numYVertices + vy) * numXEdges + ex;
1135           const PetscInt vertexL = firstVertex + (vz * numYVertices + vy) * numXVertices + ex;
1136           const PetscInt vertexR = firstVertex + nextv;
1137           PetscInt       cone[2];
1138 
1139           cone[0] = vertexL;
1140           cone[1] = vertexR;
1141           PetscCall(DMPlexSetCone(dm, edge, cone));
1142           if (dim == 2) {
1143             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1144               if (vy == numYVertices - 1) {
1145                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop));
1146                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1147                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1148                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1149               } else if (vy == 0) {
1150                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom));
1151                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1152                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1153                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1154               }
1155             } else {
1156               if (vy == 0 && cutLabel) {
1157                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1158                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1159                 if (ex == numXEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1160               }
1161             }
1162           } else {
1163             if (bdY != DM_BOUNDARY_PERIODIC) {
1164               if (vy == numYVertices - 1) {
1165                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1166                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1167                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1168               } else if (vy == 0) {
1169                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1170                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1171                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1172               }
1173             }
1174             if (bdZ != DM_BOUNDARY_PERIODIC) {
1175               if (vz == numZVertices - 1) {
1176                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1177                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1178                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1179               } else if (vz == 0) {
1180                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1181                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1182                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1183               }
1184             }
1185           }
1186         }
1187       }
1188     }
1189     PetscCall(DMPlexSymmetrize(dm));
1190     PetscCall(DMPlexStratify(dm));
1191     /* Build coordinates */
1192     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1193     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1194     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1195     PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVertices));
1196     for (v = firstVertex; v < firstVertex + numVertices; ++v) {
1197       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1198       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1199     }
1200     PetscCall(PetscSectionSetUp(coordSection));
1201     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1202     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1203     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1204     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1205     PetscCall(VecSetBlockSize(coordinates, dim));
1206     PetscCall(VecSetType(coordinates, VECSTANDARD));
1207     PetscCall(VecGetArray(coordinates, &coords));
1208     for (vz = 0; vz < numZVertices; ++vz) {
1209       for (vy = 0; vy < numYVertices; ++vy) {
1210         for (vx = 0; vx < numXVertices; ++vx) {
1211           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * vx;
1212           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * vy;
1213           if (dim == 3) coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 2] = lower[2] + ((upper[2] - lower[2]) / numZEdges) * vz;
1214         }
1215       }
1216     }
1217     PetscCall(VecRestoreArray(coordinates, &coords));
1218     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1219     PetscCall(VecDestroy(&coordinates));
1220   }
1221   PetscFunctionReturn(0);
1222 }
1223 
1224 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[]) {
1225   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1226   PetscInt       fac[3] = {0, 0, 0}, d;
1227 
1228   PetscFunctionBegin;
1229   PetscValidPointer(dm, 1);
1230   PetscValidLogicalCollectiveInt(dm, dim, 2);
1231   PetscCall(DMSetDimension(dm, dim));
1232   for (d = 0; d < dim; ++d) {
1233     fac[d] = faces[d];
1234     bdt[d] = periodicity[d];
1235   }
1236   PetscCall(DMPlexCreateCubeMesh_Internal(dm, lower, upper, fac, bdt[0], bdt[1], bdt[2]));
1237   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST || periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST || (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
1238     PetscReal L[3]       = {-1., -1., 0.};
1239     PetscReal maxCell[3] = {-1., -1., 0.};
1240 
1241     for (d = 0; d < dim; ++d) {
1242       if (periodicity[d] != DM_BOUNDARY_NONE) {
1243         L[d]       = upper[d] - lower[d];
1244         maxCell[d] = 1.1 * (L[d] / PetscMax(1, faces[d]));
1245       }
1246     }
1247     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1248   }
1249   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
1250   PetscFunctionReturn(0);
1251 }
1252 
1253 static PetscErrorCode DMPlexCreateBoxMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate) {
1254   PetscFunctionBegin;
1255   if (dim == 1) PetscCall(DMPlexCreateLineMesh_Internal(dm, faces[0], lower[0], upper[0], periodicity[0]));
1256   else if (simplex) PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1257   else PetscCall(DMPlexCreateBoxMesh_Tensor_Internal(dm, dim, faces, lower, upper, periodicity));
1258   if (!interpolate && dim > 1 && !simplex) {
1259     DM udm;
1260 
1261     PetscCall(DMPlexUninterpolate(dm, &udm));
1262     PetscCall(DMPlexCopyCoordinates(dm, udm));
1263     PetscCall(DMPlexReplace_Internal(dm, &udm));
1264   }
1265   PetscFunctionReturn(0);
1266 }
1267 
1268 /*@C
1269   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
1270 
1271   Collective
1272 
1273   Input Parameters:
1274 + comm        - The communicator for the DM object
1275 . dim         - The spatial dimension
1276 . simplex     - PETSC_TRUE for simplices, PETSC_FALSE for tensor cells
1277 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1278 . lower       - The lower left corner, or NULL for (0, 0, 0)
1279 . upper       - The upper right corner, or NULL for (1, 1, 1)
1280 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
1281 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1282 
1283   Output Parameter:
1284 . dm  - The DM object
1285 
1286   Note: If you want to customize this mesh using options, you just need to
1287 $  DMCreate(comm, &dm);
1288 $  DMSetType(dm, DMPLEX);
1289 $  DMSetFromOptions(dm);
1290 and use the options on the DMSetFromOptions() page.
1291 
1292   Here is the numbering returned for 2 faces in each direction for tensor cells:
1293 $ 10---17---11---18----12
1294 $  |         |         |
1295 $  |         |         |
1296 $ 20    2   22    3    24
1297 $  |         |         |
1298 $  |         |         |
1299 $  7---15----8---16----9
1300 $  |         |         |
1301 $  |         |         |
1302 $ 19    0   21    1   23
1303 $  |         |         |
1304 $  |         |         |
1305 $  4---13----5---14----6
1306 
1307 and for simplicial cells
1308 
1309 $ 14----8---15----9----16
1310 $  |\     5  |\      7 |
1311 $  | \       | \       |
1312 $ 13   2    14    3    15
1313 $  | 4   \   | 6   \   |
1314 $  |       \ |       \ |
1315 $ 11----6---12----7----13
1316 $  |\        |\        |
1317 $  | \    1  | \     3 |
1318 $ 10   0    11    1    12
1319 $  | 0   \   | 2   \   |
1320 $  |       \ |       \ |
1321 $  8----4----9----5----10
1322 
1323   Level: beginner
1324 
1325 .seealso: `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
1326 @*/
1327 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm) {
1328   PetscInt       fac[3] = {1, 1, 1};
1329   PetscReal      low[3] = {0, 0, 0};
1330   PetscReal      upp[3] = {1, 1, 1};
1331   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1332 
1333   PetscFunctionBegin;
1334   PetscCall(DMCreate(comm, dm));
1335   PetscCall(DMSetType(*dm, DMPLEX));
1336   PetscCall(DMPlexCreateBoxMesh_Internal(*dm, dim, simplex, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt, interpolate));
1337   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1338   PetscFunctionReturn(0);
1339 }
1340 
1341 static PetscErrorCode DMPlexCreateWedgeBoxMesh_Internal(DM dm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[]) {
1342   DM       bdm, vol;
1343   PetscInt i;
1344 
1345   PetscFunctionBegin;
1346   for (i = 0; i < 3; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity not yet supported");
1347   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &bdm));
1348   PetscCall(DMSetType(bdm, DMPLEX));
1349   PetscCall(DMSetDimension(bdm, 2));
1350   PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(bdm, 2, faces, lower, upper, periodicity, PETSC_TRUE));
1351   PetscCall(DMPlexExtrude(bdm, faces[2], upper[2] - lower[2], PETSC_TRUE, PETSC_FALSE, NULL, NULL, &vol));
1352   PetscCall(DMDestroy(&bdm));
1353   PetscCall(DMPlexReplace_Internal(dm, &vol));
1354   if (lower[2] != 0.0) {
1355     Vec          v;
1356     PetscScalar *x;
1357     PetscInt     cDim, n;
1358 
1359     PetscCall(DMGetCoordinatesLocal(dm, &v));
1360     PetscCall(VecGetBlockSize(v, &cDim));
1361     PetscCall(VecGetLocalSize(v, &n));
1362     PetscCall(VecGetArray(v, &x));
1363     x += cDim;
1364     for (i = 0; i < n; i += cDim) x[i] += lower[2];
1365     PetscCall(VecRestoreArray(v, &x));
1366     PetscCall(DMSetCoordinatesLocal(dm, v));
1367   }
1368   PetscFunctionReturn(0);
1369 }
1370 
1371 /*@
1372   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tesselating the (x,y) plane and extruding in the third direction using wedge cells.
1373 
1374   Collective
1375 
1376   Input Parameters:
1377 + comm        - The communicator for the DM object
1378 . faces       - Number of faces per dimension, or NULL for (1, 1, 1)
1379 . lower       - The lower left corner, or NULL for (0, 0, 0)
1380 . upper       - The upper right corner, or NULL for (1, 1, 1)
1381 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
1382 . orderHeight - If PETSC_TRUE, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1383 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1384 
1385   Output Parameter:
1386 . dm  - The DM object
1387 
1388   Level: beginner
1389 
1390 .seealso: `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateWedgeCylinderMesh()`, `DMExtrude()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1391 @*/
1392 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool orderHeight, PetscBool interpolate, DM *dm) {
1393   PetscInt       fac[3] = {1, 1, 1};
1394   PetscReal      low[3] = {0, 0, 0};
1395   PetscReal      upp[3] = {1, 1, 1};
1396   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1397 
1398   PetscFunctionBegin;
1399   PetscCall(DMCreate(comm, dm));
1400   PetscCall(DMSetType(*dm, DMPLEX));
1401   PetscCall(DMPlexCreateWedgeBoxMesh_Internal(*dm, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt));
1402   if (!interpolate) {
1403     DM udm;
1404 
1405     PetscCall(DMPlexUninterpolate(*dm, &udm));
1406     PetscCall(DMPlexReplace_Internal(*dm, &udm));
1407   }
1408   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1409   PetscFunctionReturn(0);
1410 }
1411 
1412 /*@C
1413   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all DM options in the database.
1414 
1415   Logically Collective on dm
1416 
1417   Input Parameters:
1418 + dm - the DM context
1419 - prefix - the prefix to prepend to all option names
1420 
1421   Notes:
1422   A hyphen (-) must NOT be given at the beginning of the prefix name.
1423   The first character of all runtime options is AUTOMATICALLY the hyphen.
1424 
1425   Level: advanced
1426 
1427 .seealso: `SNESSetFromOptions()`
1428 @*/
1429 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[]) {
1430   DM_Plex *mesh = (DM_Plex *)dm->data;
1431 
1432   PetscFunctionBegin;
1433   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1434   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, prefix));
1435   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)mesh->partitioner, prefix));
1436   PetscFunctionReturn(0);
1437 }
1438 
1439 /* Remap geometry to cylinder
1440    TODO: This only works for a single refinement, then it is broken
1441 
1442      Interior square: Linear interpolation is correct
1443      The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
1444      such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
1445 
1446        phi     = arctan(y/x)
1447        d_close = sqrt(1/8 + 1/4 sin^2(phi))
1448        d_far   = sqrt(1/2 + sin^2(phi))
1449 
1450      so we remap them using
1451 
1452        x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
1453        y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
1454 
1455      If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
1456 */
1457 static void snapToCylinder(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[]) {
1458   const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1459   const PetscReal ds2 = 0.5 * dis;
1460 
1461   if ((PetscAbsScalar(u[0]) <= ds2) && (PetscAbsScalar(u[1]) <= ds2)) {
1462     f0[0] = u[0];
1463     f0[1] = u[1];
1464   } else {
1465     PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
1466 
1467     x    = PetscRealPart(u[0]);
1468     y    = PetscRealPart(u[1]);
1469     phi  = PetscAtan2Real(y, x);
1470     sinp = PetscSinReal(phi);
1471     cosp = PetscCosReal(phi);
1472     if ((PetscAbsReal(phi) > PETSC_PI / 4.0) && (PetscAbsReal(phi) < 3.0 * PETSC_PI / 4.0)) {
1473       dc = PetscAbsReal(ds2 / sinp);
1474       df = PetscAbsReal(dis / sinp);
1475       xc = ds2 * x / PetscAbsReal(y);
1476       yc = ds2 * PetscSignReal(y);
1477     } else {
1478       dc = PetscAbsReal(ds2 / cosp);
1479       df = PetscAbsReal(dis / cosp);
1480       xc = ds2 * PetscSignReal(x);
1481       yc = ds2 * y / PetscAbsReal(x);
1482     }
1483     f0[0] = xc + (u[0] - xc) * (1.0 - dc) / (df - dc);
1484     f0[1] = yc + (u[1] - yc) * (1.0 - dc) / (df - dc);
1485   }
1486   f0[2] = u[2];
1487 }
1488 
1489 static PetscErrorCode DMPlexCreateHexCylinderMesh_Internal(DM dm, DMBoundaryType periodicZ) {
1490   const PetscInt dim = 3;
1491   PetscInt       numCells, numVertices;
1492   PetscMPIInt    rank;
1493 
1494   PetscFunctionBegin;
1495   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1496   PetscCall(DMSetDimension(dm, dim));
1497   /* Create topology */
1498   {
1499     PetscInt cone[8], c;
1500 
1501     numCells    = rank == 0 ? 5 : 0;
1502     numVertices = rank == 0 ? 16 : 0;
1503     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1504       numCells *= 3;
1505       numVertices = rank == 0 ? 24 : 0;
1506     }
1507     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1508     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 8));
1509     PetscCall(DMSetUp(dm));
1510     if (rank == 0) {
1511       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1512         cone[0] = 15;
1513         cone[1] = 18;
1514         cone[2] = 17;
1515         cone[3] = 16;
1516         cone[4] = 31;
1517         cone[5] = 32;
1518         cone[6] = 33;
1519         cone[7] = 34;
1520         PetscCall(DMPlexSetCone(dm, 0, cone));
1521         cone[0] = 16;
1522         cone[1] = 17;
1523         cone[2] = 24;
1524         cone[3] = 23;
1525         cone[4] = 32;
1526         cone[5] = 36;
1527         cone[6] = 37;
1528         cone[7] = 33; /* 22 25 26 21 */
1529         PetscCall(DMPlexSetCone(dm, 1, cone));
1530         cone[0] = 18;
1531         cone[1] = 27;
1532         cone[2] = 24;
1533         cone[3] = 17;
1534         cone[4] = 34;
1535         cone[5] = 33;
1536         cone[6] = 37;
1537         cone[7] = 38;
1538         PetscCall(DMPlexSetCone(dm, 2, cone));
1539         cone[0] = 29;
1540         cone[1] = 27;
1541         cone[2] = 18;
1542         cone[3] = 15;
1543         cone[4] = 35;
1544         cone[5] = 31;
1545         cone[6] = 34;
1546         cone[7] = 38;
1547         PetscCall(DMPlexSetCone(dm, 3, cone));
1548         cone[0] = 29;
1549         cone[1] = 15;
1550         cone[2] = 16;
1551         cone[3] = 23;
1552         cone[4] = 35;
1553         cone[5] = 36;
1554         cone[6] = 32;
1555         cone[7] = 31;
1556         PetscCall(DMPlexSetCone(dm, 4, cone));
1557 
1558         cone[0] = 31;
1559         cone[1] = 34;
1560         cone[2] = 33;
1561         cone[3] = 32;
1562         cone[4] = 19;
1563         cone[5] = 22;
1564         cone[6] = 21;
1565         cone[7] = 20;
1566         PetscCall(DMPlexSetCone(dm, 5, cone));
1567         cone[0] = 32;
1568         cone[1] = 33;
1569         cone[2] = 37;
1570         cone[3] = 36;
1571         cone[4] = 22;
1572         cone[5] = 25;
1573         cone[6] = 26;
1574         cone[7] = 21;
1575         PetscCall(DMPlexSetCone(dm, 6, cone));
1576         cone[0] = 34;
1577         cone[1] = 38;
1578         cone[2] = 37;
1579         cone[3] = 33;
1580         cone[4] = 20;
1581         cone[5] = 21;
1582         cone[6] = 26;
1583         cone[7] = 28;
1584         PetscCall(DMPlexSetCone(dm, 7, cone));
1585         cone[0] = 35;
1586         cone[1] = 38;
1587         cone[2] = 34;
1588         cone[3] = 31;
1589         cone[4] = 30;
1590         cone[5] = 19;
1591         cone[6] = 20;
1592         cone[7] = 28;
1593         PetscCall(DMPlexSetCone(dm, 8, cone));
1594         cone[0] = 35;
1595         cone[1] = 31;
1596         cone[2] = 32;
1597         cone[3] = 36;
1598         cone[4] = 30;
1599         cone[5] = 25;
1600         cone[6] = 22;
1601         cone[7] = 19;
1602         PetscCall(DMPlexSetCone(dm, 9, cone));
1603 
1604         cone[0] = 19;
1605         cone[1] = 20;
1606         cone[2] = 21;
1607         cone[3] = 22;
1608         cone[4] = 15;
1609         cone[5] = 16;
1610         cone[6] = 17;
1611         cone[7] = 18;
1612         PetscCall(DMPlexSetCone(dm, 10, cone));
1613         cone[0] = 22;
1614         cone[1] = 21;
1615         cone[2] = 26;
1616         cone[3] = 25;
1617         cone[4] = 16;
1618         cone[5] = 23;
1619         cone[6] = 24;
1620         cone[7] = 17;
1621         PetscCall(DMPlexSetCone(dm, 11, cone));
1622         cone[0] = 20;
1623         cone[1] = 28;
1624         cone[2] = 26;
1625         cone[3] = 21;
1626         cone[4] = 18;
1627         cone[5] = 17;
1628         cone[6] = 24;
1629         cone[7] = 27;
1630         PetscCall(DMPlexSetCone(dm, 12, cone));
1631         cone[0] = 30;
1632         cone[1] = 28;
1633         cone[2] = 20;
1634         cone[3] = 19;
1635         cone[4] = 29;
1636         cone[5] = 15;
1637         cone[6] = 18;
1638         cone[7] = 27;
1639         PetscCall(DMPlexSetCone(dm, 13, cone));
1640         cone[0] = 30;
1641         cone[1] = 19;
1642         cone[2] = 22;
1643         cone[3] = 25;
1644         cone[4] = 29;
1645         cone[5] = 23;
1646         cone[6] = 16;
1647         cone[7] = 15;
1648         PetscCall(DMPlexSetCone(dm, 14, cone));
1649       } else {
1650         cone[0] = 5;
1651         cone[1] = 8;
1652         cone[2] = 7;
1653         cone[3] = 6;
1654         cone[4] = 9;
1655         cone[5] = 12;
1656         cone[6] = 11;
1657         cone[7] = 10;
1658         PetscCall(DMPlexSetCone(dm, 0, cone));
1659         cone[0] = 6;
1660         cone[1] = 7;
1661         cone[2] = 14;
1662         cone[3] = 13;
1663         cone[4] = 12;
1664         cone[5] = 15;
1665         cone[6] = 16;
1666         cone[7] = 11;
1667         PetscCall(DMPlexSetCone(dm, 1, cone));
1668         cone[0] = 8;
1669         cone[1] = 17;
1670         cone[2] = 14;
1671         cone[3] = 7;
1672         cone[4] = 10;
1673         cone[5] = 11;
1674         cone[6] = 16;
1675         cone[7] = 18;
1676         PetscCall(DMPlexSetCone(dm, 2, cone));
1677         cone[0] = 19;
1678         cone[1] = 17;
1679         cone[2] = 8;
1680         cone[3] = 5;
1681         cone[4] = 20;
1682         cone[5] = 9;
1683         cone[6] = 10;
1684         cone[7] = 18;
1685         PetscCall(DMPlexSetCone(dm, 3, cone));
1686         cone[0] = 19;
1687         cone[1] = 5;
1688         cone[2] = 6;
1689         cone[3] = 13;
1690         cone[4] = 20;
1691         cone[5] = 15;
1692         cone[6] = 12;
1693         cone[7] = 9;
1694         PetscCall(DMPlexSetCone(dm, 4, cone));
1695       }
1696     }
1697     PetscCall(DMPlexSymmetrize(dm));
1698     PetscCall(DMPlexStratify(dm));
1699   }
1700   /* Create cube geometry */
1701   {
1702     Vec             coordinates;
1703     PetscSection    coordSection;
1704     PetscScalar    *coords;
1705     PetscInt        coordSize, v;
1706     const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1707     const PetscReal ds2 = dis / 2.0;
1708 
1709     /* Build coordinates */
1710     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1711     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1712     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1713     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1714     for (v = numCells; v < numCells + numVertices; ++v) {
1715       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1716       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1717     }
1718     PetscCall(PetscSectionSetUp(coordSection));
1719     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1720     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1721     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1722     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1723     PetscCall(VecSetBlockSize(coordinates, dim));
1724     PetscCall(VecSetType(coordinates, VECSTANDARD));
1725     PetscCall(VecGetArray(coordinates, &coords));
1726     if (rank == 0) {
1727       coords[0 * dim + 0]  = -ds2;
1728       coords[0 * dim + 1]  = -ds2;
1729       coords[0 * dim + 2]  = 0.0;
1730       coords[1 * dim + 0]  = ds2;
1731       coords[1 * dim + 1]  = -ds2;
1732       coords[1 * dim + 2]  = 0.0;
1733       coords[2 * dim + 0]  = ds2;
1734       coords[2 * dim + 1]  = ds2;
1735       coords[2 * dim + 2]  = 0.0;
1736       coords[3 * dim + 0]  = -ds2;
1737       coords[3 * dim + 1]  = ds2;
1738       coords[3 * dim + 2]  = 0.0;
1739       coords[4 * dim + 0]  = -ds2;
1740       coords[4 * dim + 1]  = -ds2;
1741       coords[4 * dim + 2]  = 1.0;
1742       coords[5 * dim + 0]  = -ds2;
1743       coords[5 * dim + 1]  = ds2;
1744       coords[5 * dim + 2]  = 1.0;
1745       coords[6 * dim + 0]  = ds2;
1746       coords[6 * dim + 1]  = ds2;
1747       coords[6 * dim + 2]  = 1.0;
1748       coords[7 * dim + 0]  = ds2;
1749       coords[7 * dim + 1]  = -ds2;
1750       coords[7 * dim + 2]  = 1.0;
1751       coords[8 * dim + 0]  = dis;
1752       coords[8 * dim + 1]  = -dis;
1753       coords[8 * dim + 2]  = 0.0;
1754       coords[9 * dim + 0]  = dis;
1755       coords[9 * dim + 1]  = dis;
1756       coords[9 * dim + 2]  = 0.0;
1757       coords[10 * dim + 0] = dis;
1758       coords[10 * dim + 1] = -dis;
1759       coords[10 * dim + 2] = 1.0;
1760       coords[11 * dim + 0] = dis;
1761       coords[11 * dim + 1] = dis;
1762       coords[11 * dim + 2] = 1.0;
1763       coords[12 * dim + 0] = -dis;
1764       coords[12 * dim + 1] = dis;
1765       coords[12 * dim + 2] = 0.0;
1766       coords[13 * dim + 0] = -dis;
1767       coords[13 * dim + 1] = dis;
1768       coords[13 * dim + 2] = 1.0;
1769       coords[14 * dim + 0] = -dis;
1770       coords[14 * dim + 1] = -dis;
1771       coords[14 * dim + 2] = 0.0;
1772       coords[15 * dim + 0] = -dis;
1773       coords[15 * dim + 1] = -dis;
1774       coords[15 * dim + 2] = 1.0;
1775       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1776         /* 15 31 19 */ coords[16 * dim + 0] = -ds2;
1777         coords[16 * dim + 1]                = -ds2;
1778         coords[16 * dim + 2]                = 0.5;
1779         /* 16 32 22 */ coords[17 * dim + 0] = ds2;
1780         coords[17 * dim + 1]                = -ds2;
1781         coords[17 * dim + 2]                = 0.5;
1782         /* 17 33 21 */ coords[18 * dim + 0] = ds2;
1783         coords[18 * dim + 1]                = ds2;
1784         coords[18 * dim + 2]                = 0.5;
1785         /* 18 34 20 */ coords[19 * dim + 0] = -ds2;
1786         coords[19 * dim + 1]                = ds2;
1787         coords[19 * dim + 2]                = 0.5;
1788         /* 29 35 30 */ coords[20 * dim + 0] = -dis;
1789         coords[20 * dim + 1]                = -dis;
1790         coords[20 * dim + 2]                = 0.5;
1791         /* 23 36 25 */ coords[21 * dim + 0] = dis;
1792         coords[21 * dim + 1]                = -dis;
1793         coords[21 * dim + 2]                = 0.5;
1794         /* 24 37 26 */ coords[22 * dim + 0] = dis;
1795         coords[22 * dim + 1]                = dis;
1796         coords[22 * dim + 2]                = 0.5;
1797         /* 27 38 28 */ coords[23 * dim + 0] = -dis;
1798         coords[23 * dim + 1]                = dis;
1799         coords[23 * dim + 2]                = 0.5;
1800       }
1801     }
1802     PetscCall(VecRestoreArray(coordinates, &coords));
1803     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1804     PetscCall(VecDestroy(&coordinates));
1805   }
1806   /* Create periodicity */
1807   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1808     PetscReal L[3]       = {-1., -1., 0.};
1809     PetscReal maxCell[3] = {-1., -1., 0.};
1810     PetscReal lower[3]   = {0.0, 0.0, 0.0};
1811     PetscReal upper[3]   = {1.0, 1.0, 1.5};
1812     PetscInt  numZCells  = 3;
1813 
1814     L[2]       = upper[2] - lower[2];
1815     maxCell[2] = 1.1 * (L[2] / numZCells);
1816     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1817   }
1818   {
1819     DM          cdm;
1820     PetscDS     cds;
1821     PetscScalar c[2] = {1.0, 1.0};
1822 
1823     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToCylinder));
1824     PetscCall(DMGetCoordinateDM(dm, &cdm));
1825     PetscCall(DMGetDS(cdm, &cds));
1826     PetscCall(PetscDSSetConstants(cds, 2, c));
1827   }
1828   /* Wait for coordinate creation before doing in-place modification */
1829   PetscCall(DMPlexInterpolateInPlace_Internal(dm));
1830   PetscFunctionReturn(0);
1831 }
1832 
1833 /*@
1834   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
1835 
1836   Collective
1837 
1838   Input Parameters:
1839 + comm      - The communicator for the DM object
1840 - periodicZ - The boundary type for the Z direction
1841 
1842   Output Parameter:
1843 . dm  - The DM object
1844 
1845   Note:
1846   Here is the output numbering looking from the bottom of the cylinder:
1847 $       17-----14
1848 $        |     |
1849 $        |  2  |
1850 $        |     |
1851 $ 17-----8-----7-----14
1852 $  |     |     |     |
1853 $  |  3  |  0  |  1  |
1854 $  |     |     |     |
1855 $ 19-----5-----6-----13
1856 $        |     |
1857 $        |  4  |
1858 $        |     |
1859 $       19-----13
1860 $
1861 $ and up through the top
1862 $
1863 $       18-----16
1864 $        |     |
1865 $        |  2  |
1866 $        |     |
1867 $ 18----10----11-----16
1868 $  |     |     |     |
1869 $  |  3  |  0  |  1  |
1870 $  |     |     |     |
1871 $ 20-----9----12-----15
1872 $        |     |
1873 $        |  4  |
1874 $        |     |
1875 $       20-----15
1876 
1877   Level: beginner
1878 
1879 .seealso: `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1880 @*/
1881 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, DMBoundaryType periodicZ, DM *dm) {
1882   PetscFunctionBegin;
1883   PetscValidPointer(dm, 3);
1884   PetscCall(DMCreate(comm, dm));
1885   PetscCall(DMSetType(*dm, DMPLEX));
1886   PetscCall(DMPlexCreateHexCylinderMesh_Internal(*dm, periodicZ));
1887   PetscFunctionReturn(0);
1888 }
1889 
1890 static PetscErrorCode DMPlexCreateWedgeCylinderMesh_Internal(DM dm, PetscInt n, PetscBool interpolate) {
1891   const PetscInt dim = 3;
1892   PetscInt       numCells, numVertices, v;
1893   PetscMPIInt    rank;
1894 
1895   PetscFunctionBegin;
1896   PetscCheck(n >= 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %" PetscInt_FMT " cannot be negative", n);
1897   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1898   PetscCall(DMSetDimension(dm, dim));
1899   /* Must create the celltype label here so that we do not automatically try to compute the types */
1900   PetscCall(DMCreateLabel(dm, "celltype"));
1901   /* Create topology */
1902   {
1903     PetscInt cone[6], c;
1904 
1905     numCells    = rank == 0 ? n : 0;
1906     numVertices = rank == 0 ? 2 * (n + 1) : 0;
1907     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1908     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
1909     PetscCall(DMSetUp(dm));
1910     for (c = 0; c < numCells; c++) {
1911       cone[0] = c + n * 1;
1912       cone[1] = (c + 1) % n + n * 1;
1913       cone[2] = 0 + 3 * n;
1914       cone[3] = c + n * 2;
1915       cone[4] = (c + 1) % n + n * 2;
1916       cone[5] = 1 + 3 * n;
1917       PetscCall(DMPlexSetCone(dm, c, cone));
1918       PetscCall(DMPlexSetCellType(dm, c, DM_POLYTOPE_TRI_PRISM_TENSOR));
1919     }
1920     PetscCall(DMPlexSymmetrize(dm));
1921     PetscCall(DMPlexStratify(dm));
1922   }
1923   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetCellType(dm, v, DM_POLYTOPE_POINT));
1924   /* Create cylinder geometry */
1925   {
1926     Vec          coordinates;
1927     PetscSection coordSection;
1928     PetscScalar *coords;
1929     PetscInt     coordSize, c;
1930 
1931     /* Build coordinates */
1932     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1933     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1934     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1935     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1936     for (v = numCells; v < numCells + numVertices; ++v) {
1937       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1938       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1939     }
1940     PetscCall(PetscSectionSetUp(coordSection));
1941     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1942     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1943     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1944     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1945     PetscCall(VecSetBlockSize(coordinates, dim));
1946     PetscCall(VecSetType(coordinates, VECSTANDARD));
1947     PetscCall(VecGetArray(coordinates, &coords));
1948     for (c = 0; c < numCells; c++) {
1949       coords[(c + 0 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
1950       coords[(c + 0 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
1951       coords[(c + 0 * n) * dim + 2] = 1.0;
1952       coords[(c + 1 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
1953       coords[(c + 1 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
1954       coords[(c + 1 * n) * dim + 2] = 0.0;
1955     }
1956     if (rank == 0) {
1957       coords[(2 * n + 0) * dim + 0] = 0.0;
1958       coords[(2 * n + 0) * dim + 1] = 0.0;
1959       coords[(2 * n + 0) * dim + 2] = 1.0;
1960       coords[(2 * n + 1) * dim + 0] = 0.0;
1961       coords[(2 * n + 1) * dim + 1] = 0.0;
1962       coords[(2 * n + 1) * dim + 2] = 0.0;
1963     }
1964     PetscCall(VecRestoreArray(coordinates, &coords));
1965     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1966     PetscCall(VecDestroy(&coordinates));
1967   }
1968   /* Interpolate */
1969   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
1970   PetscFunctionReturn(0);
1971 }
1972 
1973 /*@
1974   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
1975 
1976   Collective
1977 
1978   Input Parameters:
1979 + comm - The communicator for the DM object
1980 . n    - The number of wedges around the origin
1981 - interpolate - Create edges and faces
1982 
1983   Output Parameter:
1984 . dm  - The DM object
1985 
1986   Level: beginner
1987 
1988 .seealso: `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1989 @*/
1990 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm) {
1991   PetscFunctionBegin;
1992   PetscValidPointer(dm, 4);
1993   PetscCall(DMCreate(comm, dm));
1994   PetscCall(DMSetType(*dm, DMPLEX));
1995   PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(*dm, n, interpolate));
1996   PetscFunctionReturn(0);
1997 }
1998 
1999 static inline PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[]) {
2000   PetscReal prod = 0.0;
2001   PetscInt  i;
2002   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
2003   return PetscSqrtReal(prod);
2004 }
2005 static inline PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[]) {
2006   PetscReal prod = 0.0;
2007   PetscInt  i;
2008   for (i = 0; i < dim; ++i) prod += x[i] * y[i];
2009   return prod;
2010 }
2011 
2012 /* The first constant is the sphere radius */
2013 static void snapToSphere(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[]) {
2014   PetscReal r     = PetscRealPart(constants[0]);
2015   PetscReal norm2 = 0.0, fac;
2016   PetscInt  n     = uOff[1] - uOff[0], d;
2017 
2018   for (d = 0; d < n; ++d) norm2 += PetscSqr(PetscRealPart(u[d]));
2019   fac = r / PetscSqrtReal(norm2);
2020   for (d = 0; d < n; ++d) f0[d] = u[d] * fac;
2021 }
2022 
2023 static PetscErrorCode DMPlexCreateSphereMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, PetscReal R) {
2024   const PetscInt embedDim = dim + 1;
2025   PetscSection   coordSection;
2026   Vec            coordinates;
2027   PetscScalar   *coords;
2028   PetscReal     *coordsIn;
2029   PetscInt       numCells, numEdges, numVerts = 0, firstVertex = 0, v, firstEdge, coordSize, d, c, e;
2030   PetscMPIInt    rank;
2031 
2032   PetscFunctionBegin;
2033   PetscValidLogicalCollectiveBool(dm, simplex, 3);
2034   PetscCall(DMSetDimension(dm, dim));
2035   PetscCall(DMSetCoordinateDim(dm, dim + 1));
2036   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2037   switch (dim) {
2038   case 2:
2039     if (simplex) {
2040       const PetscReal radius    = PetscSqrtReal(1 + PETSC_PHI * PETSC_PHI) / (1.0 + PETSC_PHI);
2041       const PetscReal edgeLen   = 2.0 / (1.0 + PETSC_PHI) * (R / radius);
2042       const PetscInt  degree    = 5;
2043       PetscReal       vertex[3] = {0.0, 1.0 / (1.0 + PETSC_PHI), PETSC_PHI / (1.0 + PETSC_PHI)};
2044       PetscInt        s[3]      = {1, 1, 1};
2045       PetscInt        cone[3];
2046       PetscInt       *graph, p, i, j, k;
2047 
2048       vertex[0] *= R / radius;
2049       vertex[1] *= R / radius;
2050       vertex[2] *= R / radius;
2051       numCells    = rank == 0 ? 20 : 0;
2052       numVerts    = rank == 0 ? 12 : 0;
2053       firstVertex = numCells;
2054       /* Use icosahedron, which for a R-sphere has coordinates which are all cyclic permutations of
2055 
2056            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
2057 
2058          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2059          length is then given by 2/(1+\phi) = 2 * 0.38197 = 0.76393.
2060       */
2061       /* Construct vertices */
2062       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2063       if (rank == 0) {
2064         for (p = 0, i = 0; p < embedDim; ++p) {
2065           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2066             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2067               for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertex[(d + p) % embedDim];
2068               ++i;
2069             }
2070           }
2071         }
2072       }
2073       /* Construct graph */
2074       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2075       for (i = 0; i < numVerts; ++i) {
2076         for (j = 0, k = 0; j < numVerts; ++j) {
2077           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2078             graph[i * numVerts + j] = 1;
2079             ++k;
2080           }
2081         }
2082         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid icosahedron, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2083       }
2084       /* Build Topology */
2085       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2086       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2087       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2088       /* Cells */
2089       for (i = 0, c = 0; i < numVerts; ++i) {
2090         for (j = 0; j < i; ++j) {
2091           for (k = 0; k < j; ++k) {
2092             if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i]) {
2093               cone[0] = firstVertex + i;
2094               cone[1] = firstVertex + j;
2095               cone[2] = firstVertex + k;
2096               /* Check orientation */
2097               {
2098                 const PetscInt epsilon[3][3][3] = {
2099                   {{0, 0, 0},  {0, 0, 1},  {0, -1, 0}},
2100                   {{0, 0, -1}, {0, 0, 0},  {1, 0, 0} },
2101                   {{0, 1, 0},  {-1, 0, 0}, {0, 0, 0} }
2102                 };
2103                 PetscReal normal[3];
2104                 PetscInt  e, f;
2105 
2106                 for (d = 0; d < embedDim; ++d) {
2107                   normal[d] = 0.0;
2108                   for (e = 0; e < embedDim; ++e) {
2109                     for (f = 0; f < embedDim; ++f) normal[d] += epsilon[d][e][f] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]);
2110                   }
2111                 }
2112                 if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2113                   PetscInt tmp = cone[1];
2114                   cone[1]      = cone[2];
2115                   cone[2]      = tmp;
2116                 }
2117               }
2118               PetscCall(DMPlexSetCone(dm, c++, cone));
2119             }
2120           }
2121         }
2122       }
2123       PetscCall(DMPlexSymmetrize(dm));
2124       PetscCall(DMPlexStratify(dm));
2125       PetscCall(PetscFree(graph));
2126     } else {
2127       /*
2128         12-21--13
2129          |     |
2130         25  4  24
2131          |     |
2132   12-25--9-16--8-24--13
2133    |     |     |     |
2134   23  5 17  0 15  3  22
2135    |     |     |     |
2136   10-20--6-14--7-19--11
2137          |     |
2138         20  1  19
2139          |     |
2140         10-18--11
2141          |     |
2142         23  2  22
2143          |     |
2144         12-21--13
2145        */
2146       PetscInt cone[4], ornt[4];
2147 
2148       numCells    = rank == 0 ? 6 : 0;
2149       numEdges    = rank == 0 ? 12 : 0;
2150       numVerts    = rank == 0 ? 8 : 0;
2151       firstVertex = numCells;
2152       firstEdge   = numCells + numVerts;
2153       /* Build Topology */
2154       PetscCall(DMPlexSetChart(dm, 0, numCells + numEdges + numVerts));
2155       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 4));
2156       for (e = firstEdge; e < firstEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
2157       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2158       if (rank == 0) {
2159         /* Cell 0 */
2160         cone[0] = 14;
2161         cone[1] = 15;
2162         cone[2] = 16;
2163         cone[3] = 17;
2164         PetscCall(DMPlexSetCone(dm, 0, cone));
2165         ornt[0] = 0;
2166         ornt[1] = 0;
2167         ornt[2] = 0;
2168         ornt[3] = 0;
2169         PetscCall(DMPlexSetConeOrientation(dm, 0, ornt));
2170         /* Cell 1 */
2171         cone[0] = 18;
2172         cone[1] = 19;
2173         cone[2] = 14;
2174         cone[3] = 20;
2175         PetscCall(DMPlexSetCone(dm, 1, cone));
2176         ornt[0] = 0;
2177         ornt[1] = 0;
2178         ornt[2] = -1;
2179         ornt[3] = 0;
2180         PetscCall(DMPlexSetConeOrientation(dm, 1, ornt));
2181         /* Cell 2 */
2182         cone[0] = 21;
2183         cone[1] = 22;
2184         cone[2] = 18;
2185         cone[3] = 23;
2186         PetscCall(DMPlexSetCone(dm, 2, cone));
2187         ornt[0] = 0;
2188         ornt[1] = 0;
2189         ornt[2] = -1;
2190         ornt[3] = 0;
2191         PetscCall(DMPlexSetConeOrientation(dm, 2, ornt));
2192         /* Cell 3 */
2193         cone[0] = 19;
2194         cone[1] = 22;
2195         cone[2] = 24;
2196         cone[3] = 15;
2197         PetscCall(DMPlexSetCone(dm, 3, cone));
2198         ornt[0] = -1;
2199         ornt[1] = -1;
2200         ornt[2] = 0;
2201         ornt[3] = -1;
2202         PetscCall(DMPlexSetConeOrientation(dm, 3, ornt));
2203         /* Cell 4 */
2204         cone[0] = 16;
2205         cone[1] = 24;
2206         cone[2] = 21;
2207         cone[3] = 25;
2208         PetscCall(DMPlexSetCone(dm, 4, cone));
2209         ornt[0] = -1;
2210         ornt[1] = -1;
2211         ornt[2] = -1;
2212         ornt[3] = 0;
2213         PetscCall(DMPlexSetConeOrientation(dm, 4, ornt));
2214         /* Cell 5 */
2215         cone[0] = 20;
2216         cone[1] = 17;
2217         cone[2] = 25;
2218         cone[3] = 23;
2219         PetscCall(DMPlexSetCone(dm, 5, cone));
2220         ornt[0] = -1;
2221         ornt[1] = -1;
2222         ornt[2] = -1;
2223         ornt[3] = -1;
2224         PetscCall(DMPlexSetConeOrientation(dm, 5, ornt));
2225         /* Edges */
2226         cone[0] = 6;
2227         cone[1] = 7;
2228         PetscCall(DMPlexSetCone(dm, 14, cone));
2229         cone[0] = 7;
2230         cone[1] = 8;
2231         PetscCall(DMPlexSetCone(dm, 15, cone));
2232         cone[0] = 8;
2233         cone[1] = 9;
2234         PetscCall(DMPlexSetCone(dm, 16, cone));
2235         cone[0] = 9;
2236         cone[1] = 6;
2237         PetscCall(DMPlexSetCone(dm, 17, cone));
2238         cone[0] = 10;
2239         cone[1] = 11;
2240         PetscCall(DMPlexSetCone(dm, 18, cone));
2241         cone[0] = 11;
2242         cone[1] = 7;
2243         PetscCall(DMPlexSetCone(dm, 19, cone));
2244         cone[0] = 6;
2245         cone[1] = 10;
2246         PetscCall(DMPlexSetCone(dm, 20, cone));
2247         cone[0] = 12;
2248         cone[1] = 13;
2249         PetscCall(DMPlexSetCone(dm, 21, cone));
2250         cone[0] = 13;
2251         cone[1] = 11;
2252         PetscCall(DMPlexSetCone(dm, 22, cone));
2253         cone[0] = 10;
2254         cone[1] = 12;
2255         PetscCall(DMPlexSetCone(dm, 23, cone));
2256         cone[0] = 13;
2257         cone[1] = 8;
2258         PetscCall(DMPlexSetCone(dm, 24, cone));
2259         cone[0] = 12;
2260         cone[1] = 9;
2261         PetscCall(DMPlexSetCone(dm, 25, cone));
2262       }
2263       PetscCall(DMPlexSymmetrize(dm));
2264       PetscCall(DMPlexStratify(dm));
2265       /* Build coordinates */
2266       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2267       if (rank == 0) {
2268         coordsIn[0 * embedDim + 0] = -R;
2269         coordsIn[0 * embedDim + 1] = R;
2270         coordsIn[0 * embedDim + 2] = -R;
2271         coordsIn[1 * embedDim + 0] = R;
2272         coordsIn[1 * embedDim + 1] = R;
2273         coordsIn[1 * embedDim + 2] = -R;
2274         coordsIn[2 * embedDim + 0] = R;
2275         coordsIn[2 * embedDim + 1] = -R;
2276         coordsIn[2 * embedDim + 2] = -R;
2277         coordsIn[3 * embedDim + 0] = -R;
2278         coordsIn[3 * embedDim + 1] = -R;
2279         coordsIn[3 * embedDim + 2] = -R;
2280         coordsIn[4 * embedDim + 0] = -R;
2281         coordsIn[4 * embedDim + 1] = R;
2282         coordsIn[4 * embedDim + 2] = R;
2283         coordsIn[5 * embedDim + 0] = R;
2284         coordsIn[5 * embedDim + 1] = R;
2285         coordsIn[5 * embedDim + 2] = R;
2286         coordsIn[6 * embedDim + 0] = -R;
2287         coordsIn[6 * embedDim + 1] = -R;
2288         coordsIn[6 * embedDim + 2] = R;
2289         coordsIn[7 * embedDim + 0] = R;
2290         coordsIn[7 * embedDim + 1] = -R;
2291         coordsIn[7 * embedDim + 2] = R;
2292       }
2293     }
2294     break;
2295   case 3:
2296     if (simplex) {
2297       const PetscReal edgeLen         = 1.0 / PETSC_PHI;
2298       PetscReal       vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
2299       PetscReal       vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
2300       PetscReal       vertexC[4]      = {0.5, 0.5 * PETSC_PHI, 0.5 / PETSC_PHI, 0.0};
2301       const PetscInt  degree          = 12;
2302       PetscInt        s[4]            = {1, 1, 1};
2303       PetscInt        evenPerm[12][4] = {
2304                {0, 1, 2, 3},
2305                {0, 2, 3, 1},
2306                {0, 3, 1, 2},
2307                {1, 0, 3, 2},
2308                {1, 2, 0, 3},
2309                {1, 3, 2, 0},
2310                {2, 0, 1, 3},
2311                {2, 1, 3, 0},
2312                {2, 3, 0, 1},
2313                {3, 0, 2, 1},
2314                {3, 1, 0, 2},
2315                {3, 2, 1, 0}
2316       };
2317       PetscInt  cone[4];
2318       PetscInt *graph, p, i, j, k, l;
2319 
2320       vertexA[0] *= R;
2321       vertexA[1] *= R;
2322       vertexA[2] *= R;
2323       vertexA[3] *= R;
2324       vertexB[0] *= R;
2325       vertexB[1] *= R;
2326       vertexB[2] *= R;
2327       vertexB[3] *= R;
2328       vertexC[0] *= R;
2329       vertexC[1] *= R;
2330       vertexC[2] *= R;
2331       vertexC[3] *= R;
2332       numCells    = rank == 0 ? 600 : 0;
2333       numVerts    = rank == 0 ? 120 : 0;
2334       firstVertex = numCells;
2335       /* Use the 600-cell, which for a unit sphere has coordinates which are
2336 
2337            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
2338                (\pm 1,    0,       0,      0)  all cyclic permutations   8
2339            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
2340 
2341          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2342          length is then given by 1/\phi = 0.61803.
2343 
2344          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
2345          http://mathworld.wolfram.com/600-Cell.html
2346       */
2347       /* Construct vertices */
2348       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2349       i = 0;
2350       if (rank == 0) {
2351         for (s[0] = -1; s[0] < 2; s[0] += 2) {
2352           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2353             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2354               for (s[3] = -1; s[3] < 2; s[3] += 2) {
2355                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[d] * vertexA[d];
2356                 ++i;
2357               }
2358             }
2359           }
2360         }
2361         for (p = 0; p < embedDim; ++p) {
2362           s[1] = s[2] = s[3] = 1;
2363           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2364             for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertexB[(d + p) % embedDim];
2365             ++i;
2366           }
2367         }
2368         for (p = 0; p < 12; ++p) {
2369           s[3] = 1;
2370           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2371             for (s[1] = -1; s[1] < 2; s[1] += 2) {
2372               for (s[2] = -1; s[2] < 2; s[2] += 2) {
2373                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[evenPerm[p][d]] * vertexC[evenPerm[p][d]];
2374                 ++i;
2375               }
2376             }
2377           }
2378         }
2379       }
2380       PetscCheck(i == numVerts, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertices %" PetscInt_FMT " != %" PetscInt_FMT, i, numVerts);
2381       /* Construct graph */
2382       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2383       for (i = 0; i < numVerts; ++i) {
2384         for (j = 0, k = 0; j < numVerts; ++j) {
2385           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2386             graph[i * numVerts + j] = 1;
2387             ++k;
2388           }
2389         }
2390         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2391       }
2392       /* Build Topology */
2393       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2394       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2395       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2396       /* Cells */
2397       if (rank == 0) {
2398         for (i = 0, c = 0; i < numVerts; ++i) {
2399           for (j = 0; j < i; ++j) {
2400             for (k = 0; k < j; ++k) {
2401               for (l = 0; l < k; ++l) {
2402                 if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i] && graph[l * numVerts + i] && graph[l * numVerts + j] && graph[l * numVerts + k]) {
2403                   cone[0] = firstVertex + i;
2404                   cone[1] = firstVertex + j;
2405                   cone[2] = firstVertex + k;
2406                   cone[3] = firstVertex + l;
2407                   /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
2408                   {
2409                     const PetscInt epsilon[4][4][4][4] = {
2410                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, -1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 0, 0}, {0, 1, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 1, 0}, {0, -1, 0, 0}, {0, 0, 0, 0}}},
2411 
2412                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}}, {{0, 0, -1, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}}},
2413 
2414                       {{{0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, 0, 0}, {0, -1, 0, 0}}, {{0, 0, 0, -1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 1, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}},
2415 
2416                       {{{0, 0, 0, 0}, {0, 0, -1, 0}, {0, 1, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 1, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}}, {{0, -1, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}} }
2417                     };
2418                     PetscReal normal[4];
2419                     PetscInt  e, f, g;
2420 
2421                     for (d = 0; d < embedDim; ++d) {
2422                       normal[d] = 0.0;
2423                       for (e = 0; e < embedDim; ++e) {
2424                         for (f = 0; f < embedDim; ++f) {
2425                           for (g = 0; g < embedDim; ++g) {
2426                             normal[d] += epsilon[d][e][f][g] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]) * (coordsIn[l * embedDim + f] - coordsIn[i * embedDim + f]);
2427                           }
2428                         }
2429                       }
2430                     }
2431                     if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2432                       PetscInt tmp = cone[1];
2433                       cone[1]      = cone[2];
2434                       cone[2]      = tmp;
2435                     }
2436                   }
2437                   PetscCall(DMPlexSetCone(dm, c++, cone));
2438                 }
2439               }
2440             }
2441           }
2442         }
2443       }
2444       PetscCall(DMPlexSymmetrize(dm));
2445       PetscCall(DMPlexStratify(dm));
2446       PetscCall(PetscFree(graph));
2447     }
2448     break;
2449   default: SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unsupported dimension for sphere: %" PetscInt_FMT, dim);
2450   }
2451   /* Create coordinates */
2452   PetscCall(DMGetCoordinateSection(dm, &coordSection));
2453   PetscCall(PetscSectionSetNumFields(coordSection, 1));
2454   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, embedDim));
2455   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVerts));
2456   for (v = firstVertex; v < firstVertex + numVerts; ++v) {
2457     PetscCall(PetscSectionSetDof(coordSection, v, embedDim));
2458     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, embedDim));
2459   }
2460   PetscCall(PetscSectionSetUp(coordSection));
2461   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2462   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2463   PetscCall(VecSetBlockSize(coordinates, embedDim));
2464   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2465   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2466   PetscCall(VecSetType(coordinates, VECSTANDARD));
2467   PetscCall(VecGetArray(coordinates, &coords));
2468   for (v = 0; v < numVerts; ++v)
2469     for (d = 0; d < embedDim; ++d) coords[v * embedDim + d] = coordsIn[v * embedDim + d];
2470   PetscCall(VecRestoreArray(coordinates, &coords));
2471   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2472   PetscCall(VecDestroy(&coordinates));
2473   PetscCall(PetscFree(coordsIn));
2474   {
2475     DM          cdm;
2476     PetscDS     cds;
2477     PetscScalar c = R;
2478 
2479     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToSphere));
2480     PetscCall(DMGetCoordinateDM(dm, &cdm));
2481     PetscCall(DMGetDS(cdm, &cds));
2482     PetscCall(PetscDSSetConstants(cds, 1, &c));
2483   }
2484   /* Wait for coordinate creation before doing in-place modification */
2485   if (simplex) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2486   PetscFunctionReturn(0);
2487 }
2488 
2489 typedef void (*TPSEvaluateFunc)(const PetscReal[], PetscReal *, PetscReal[], PetscReal (*)[3]);
2490 
2491 /*
2492  The Schwarz P implicit surface is
2493 
2494      f(x) = cos(x0) + cos(x1) + cos(x2) = 0
2495 */
2496 static void TPSEvaluate_SchwarzP(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3]) {
2497   PetscReal c[3] = {PetscCosReal(y[0] * PETSC_PI), PetscCosReal(y[1] * PETSC_PI), PetscCosReal(y[2] * PETSC_PI)};
2498   PetscReal g[3] = {-PetscSinReal(y[0] * PETSC_PI), -PetscSinReal(y[1] * PETSC_PI), -PetscSinReal(y[2] * PETSC_PI)};
2499   f[0]           = c[0] + c[1] + c[2];
2500   for (PetscInt i = 0; i < 3; i++) {
2501     grad[i] = PETSC_PI * g[i];
2502     for (PetscInt j = 0; j < 3; j++) hess[i][j] = (i == j) ? -PetscSqr(PETSC_PI) * c[i] : 0.;
2503   }
2504 }
2505 
2506 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2507 static PetscErrorCode TPSExtrudeNormalFunc_SchwarzP(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx) {
2508   for (PetscInt i = 0; i < 3; i++) u[i] = -PETSC_PI * PetscSinReal(x[i] * PETSC_PI);
2509   return 0;
2510 }
2511 
2512 /*
2513  The Gyroid implicit surface is
2514 
2515  f(x,y,z) = sin(pi * x) * cos (pi * (y + 1/2))  + sin(pi * (y + 1/2)) * cos(pi * (z + 1/4)) + sin(pi * (z + 1/4)) * cos(pi * x)
2516 
2517 */
2518 static void TPSEvaluate_Gyroid(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3]) {
2519   PetscReal s[3] = {PetscSinReal(PETSC_PI * y[0]), PetscSinReal(PETSC_PI * (y[1] + .5)), PetscSinReal(PETSC_PI * (y[2] + .25))};
2520   PetscReal c[3] = {PetscCosReal(PETSC_PI * y[0]), PetscCosReal(PETSC_PI * (y[1] + .5)), PetscCosReal(PETSC_PI * (y[2] + .25))};
2521   f[0]           = s[0] * c[1] + s[1] * c[2] + s[2] * c[0];
2522   grad[0]        = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2523   grad[1]        = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2524   grad[2]        = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2525   hess[0][0]     = -PetscSqr(PETSC_PI) * (s[0] * c[1] + s[2] * c[0]);
2526   hess[0][1]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2527   hess[0][2]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2528   hess[1][0]     = -PetscSqr(PETSC_PI) * (s[1] * c[2] + s[0] * c[1]);
2529   hess[1][1]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2530   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2531   hess[2][0]     = -PetscSqr(PETSC_PI) * (s[2] * c[0] + s[1] * c[2]);
2532   hess[2][1]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2533   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2534 }
2535 
2536 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2537 static PetscErrorCode TPSExtrudeNormalFunc_Gyroid(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx) {
2538   PetscReal s[3] = {PetscSinReal(PETSC_PI * x[0]), PetscSinReal(PETSC_PI * (x[1] + .5)), PetscSinReal(PETSC_PI * (x[2] + .25))};
2539   PetscReal c[3] = {PetscCosReal(PETSC_PI * x[0]), PetscCosReal(PETSC_PI * (x[1] + .5)), PetscCosReal(PETSC_PI * (x[2] + .25))};
2540   u[0]           = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2541   u[1]           = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2542   u[2]           = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2543   return 0;
2544 }
2545 
2546 /*
2547    We wish to solve
2548 
2549          min_y || y - x ||^2  subject to f(y) = 0
2550 
2551    Let g(y) = grad(f).  The minimization problem is equivalent to asking to satisfy
2552    f(y) = 0 and (y-x) is parallel to g(y).  We do this by using Householder QR to obtain a basis for the
2553    tangent space and ask for both components in the tangent space to be zero.
2554 
2555    Take g to be a column vector and compute the "full QR" factorization Q R = g,
2556    where Q = I - 2 n n^T is a symmetric orthogonal matrix.
2557    The first column of Q is parallel to g so the remaining two columns span the null space.
2558    Let Qn = Q[:,1:] be those remaining columns.  Then Qn Qn^T is an orthogonal projector into the tangent space.
2559    Since Q is symmetric, this is equivalent to multipyling by Q and taking the last two entries.
2560    In total, we have a system of 3 equations in 3 unknowns:
2561 
2562      f(y) = 0                       1 equation
2563      Qn^T (y - x) = 0               2 equations
2564 
2565    Here, we compute the residual and Jacobian of this system.
2566 */
2567 static void TPSNearestPointResJac(TPSEvaluateFunc feval, const PetscScalar x[], const PetscScalar y[], PetscScalar res[], PetscScalar J[]) {
2568   PetscReal yreal[3] = {PetscRealPart(y[0]), PetscRealPart(y[1]), PetscRealPart(y[2])};
2569   PetscReal d[3]     = {PetscRealPart(y[0] - x[0]), PetscRealPart(y[1] - x[1]), PetscRealPart(y[2] - x[2])};
2570   PetscReal f, grad[3], n[3], norm, norm_y[3], nd, nd_y[3], sign;
2571   PetscReal n_y[3][3] = {
2572     {0, 0, 0},
2573     {0, 0, 0},
2574     {0, 0, 0}
2575   };
2576 
2577   feval(yreal, &f, grad, n_y);
2578 
2579   for (PetscInt i = 0; i < 3; i++) n[i] = grad[i];
2580   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2581   for (PetscInt i = 0; i < 3; i++) norm_y[i] = 1. / norm * n[i] * n_y[i][i];
2582 
2583   // Define the Householder reflector
2584   sign = n[0] >= 0 ? 1. : -1.;
2585   n[0] += norm * sign;
2586   for (PetscInt i = 0; i < 3; i++) n_y[0][i] += norm_y[i] * sign;
2587 
2588   norm      = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2589   norm_y[0] = 1. / norm * (n[0] * n_y[0][0]);
2590   norm_y[1] = 1. / norm * (n[0] * n_y[0][1] + n[1] * n_y[1][1]);
2591   norm_y[2] = 1. / norm * (n[0] * n_y[0][2] + n[2] * n_y[2][2]);
2592 
2593   for (PetscInt i = 0; i < 3; i++) {
2594     n[i] /= norm;
2595     for (PetscInt j = 0; j < 3; j++) {
2596       // note that n[i] is n_old[i]/norm when executing the code below
2597       n_y[i][j] = n_y[i][j] / norm - n[i] / norm * norm_y[j];
2598     }
2599   }
2600 
2601   nd = n[0] * d[0] + n[1] * d[1] + n[2] * d[2];
2602   for (PetscInt i = 0; i < 3; i++) nd_y[i] = n[i] + n_y[0][i] * d[0] + n_y[1][i] * d[1] + n_y[2][i] * d[2];
2603 
2604   res[0] = f;
2605   res[1] = d[1] - 2 * n[1] * nd;
2606   res[2] = d[2] - 2 * n[2] * nd;
2607   // J[j][i] is J_{ij} (column major)
2608   for (PetscInt j = 0; j < 3; j++) {
2609     J[0 + j * 3] = grad[j];
2610     J[1 + j * 3] = (j == 1) * 1. - 2 * (n_y[1][j] * nd + n[1] * nd_y[j]);
2611     J[2 + j * 3] = (j == 2) * 1. - 2 * (n_y[2][j] * nd + n[2] * nd_y[j]);
2612   }
2613 }
2614 
2615 /*
2616    Project x to the nearest point on the implicit surface using Newton's method.
2617 */
2618 static PetscErrorCode TPSNearestPoint(TPSEvaluateFunc feval, PetscScalar x[]) {
2619   PetscScalar y[3] = {x[0], x[1], x[2]}; // Initial guess
2620 
2621   PetscFunctionBegin;
2622   for (PetscInt iter = 0; iter < 10; iter++) {
2623     PetscScalar res[3], J[9];
2624     PetscReal   resnorm;
2625     TPSNearestPointResJac(feval, x, y, res, J);
2626     resnorm = PetscSqrtReal(PetscSqr(PetscRealPart(res[0])) + PetscSqr(PetscRealPart(res[1])) + PetscSqr(PetscRealPart(res[2])));
2627     if (0) { // Turn on this monitor if you need to confirm quadratic convergence
2628       PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%" PetscInt_FMT "] res [%g %g %g]\n", iter, (double)PetscRealPart(res[0]), (double)PetscRealPart(res[1]), (double)PetscRealPart(res[2])));
2629     }
2630     if (resnorm < PETSC_SMALL) break;
2631 
2632     // Take the Newton step
2633     PetscCall(PetscKernel_A_gets_inverse_A_3(J, 0., PETSC_FALSE, NULL));
2634     PetscKernel_v_gets_v_minus_A_times_w_3(y, J, res);
2635   }
2636   for (PetscInt i = 0; i < 3; i++) x[i] = y[i];
2637   PetscFunctionReturn(0);
2638 }
2639 
2640 const char *const DMPlexTPSTypes[] = {"SCHWARZ_P", "GYROID", "DMPlexTPSType", "DMPLEX_TPS_", NULL};
2641 
2642 static PetscErrorCode DMPlexCreateTPSMesh_Internal(DM dm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness) {
2643   PetscMPIInt rank;
2644   PetscInt    topoDim = 2, spaceDim = 3, numFaces = 0, numVertices = 0, numEdges = 0;
2645   PetscInt(*edges)[2] = NULL, *edgeSets = NULL;
2646   PetscInt            *cells_flat = NULL;
2647   PetscReal           *vtxCoords  = NULL;
2648   TPSEvaluateFunc      evalFunc   = NULL;
2649   PetscSimplePointFunc normalFunc = NULL;
2650   DMLabel              label;
2651 
2652   PetscFunctionBegin;
2653   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2654   PetscCheck((layers != 0) ^ (thickness == 0.), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Layers %" PetscInt_FMT " must be nonzero iff thickness %g is nonzero", layers, (double)thickness);
2655   switch (tpstype) {
2656   case DMPLEX_TPS_SCHWARZ_P:
2657     PetscCheck(!periodic || (periodic[0] == DM_BOUNDARY_NONE && periodic[1] == DM_BOUNDARY_NONE && periodic[2] == DM_BOUNDARY_NONE), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Schwarz P does not support periodic meshes");
2658     if (rank == 0) {
2659       PetscInt(*cells)[6][4][4] = NULL; // [junction, junction-face, cell, conn]
2660       PetscInt  Njunctions = 0, Ncuts = 0, Npipes[3], vcount;
2661       PetscReal L = 1;
2662 
2663       Npipes[0]   = (extent[0] + 1) * extent[1] * extent[2];
2664       Npipes[1]   = extent[0] * (extent[1] + 1) * extent[2];
2665       Npipes[2]   = extent[0] * extent[1] * (extent[2] + 1);
2666       Njunctions  = extent[0] * extent[1] * extent[2];
2667       Ncuts       = 2 * (extent[0] * extent[1] + extent[1] * extent[2] + extent[2] * extent[0]);
2668       numVertices = 4 * (Npipes[0] + Npipes[1] + Npipes[2]) + 8 * Njunctions;
2669       PetscCall(PetscMalloc1(3 * numVertices, &vtxCoords));
2670       PetscCall(PetscMalloc1(Njunctions, &cells));
2671       PetscCall(PetscMalloc1(Ncuts * 4, &edges));
2672       PetscCall(PetscMalloc1(Ncuts * 4, &edgeSets));
2673       // x-normal pipes
2674       vcount = 0;
2675       for (PetscInt i = 0; i < extent[0] + 1; i++) {
2676         for (PetscInt j = 0; j < extent[1]; j++) {
2677           for (PetscInt k = 0; k < extent[2]; k++) {
2678             for (PetscInt l = 0; l < 4; l++) {
2679               vtxCoords[vcount++] = (2 * i - 1) * L;
2680               vtxCoords[vcount++] = 2 * j * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2681               vtxCoords[vcount++] = 2 * k * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2682             }
2683           }
2684         }
2685       }
2686       // y-normal pipes
2687       for (PetscInt i = 0; i < extent[0]; i++) {
2688         for (PetscInt j = 0; j < extent[1] + 1; j++) {
2689           for (PetscInt k = 0; k < extent[2]; k++) {
2690             for (PetscInt l = 0; l < 4; l++) {
2691               vtxCoords[vcount++] = 2 * i * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2692               vtxCoords[vcount++] = (2 * j - 1) * L;
2693               vtxCoords[vcount++] = 2 * k * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2694             }
2695           }
2696         }
2697       }
2698       // z-normal pipes
2699       for (PetscInt i = 0; i < extent[0]; i++) {
2700         for (PetscInt j = 0; j < extent[1]; j++) {
2701           for (PetscInt k = 0; k < extent[2] + 1; k++) {
2702             for (PetscInt l = 0; l < 4; l++) {
2703               vtxCoords[vcount++] = 2 * i * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2704               vtxCoords[vcount++] = 2 * j * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2705               vtxCoords[vcount++] = (2 * k - 1) * L;
2706             }
2707           }
2708         }
2709       }
2710       // junctions
2711       for (PetscInt i = 0; i < extent[0]; i++) {
2712         for (PetscInt j = 0; j < extent[1]; j++) {
2713           for (PetscInt k = 0; k < extent[2]; k++) {
2714             const PetscInt J = (i * extent[1] + j) * extent[2] + k, Jvoff = (Npipes[0] + Npipes[1] + Npipes[2]) * 4 + J * 8;
2715             PetscCheck(vcount / 3 == Jvoff, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected vertex count");
2716             for (PetscInt ii = 0; ii < 2; ii++) {
2717               for (PetscInt jj = 0; jj < 2; jj++) {
2718                 for (PetscInt kk = 0; kk < 2; kk++) {
2719                   double Ls           = (1 - sqrt(2) / 4) * L;
2720                   vtxCoords[vcount++] = 2 * i * L + (2 * ii - 1) * Ls;
2721                   vtxCoords[vcount++] = 2 * j * L + (2 * jj - 1) * Ls;
2722                   vtxCoords[vcount++] = 2 * k * L + (2 * kk - 1) * Ls;
2723                 }
2724               }
2725             }
2726             const PetscInt jfaces[3][2][4] = {
2727               {{3, 1, 0, 2}, {7, 5, 4, 6}}, // x-aligned
2728               {{5, 4, 0, 1}, {7, 6, 2, 3}}, // y-aligned
2729               {{6, 2, 0, 4}, {7, 3, 1, 5}}  // z-aligned
2730             };
2731             const PetscInt pipe_lo[3] = {// vertex numbers of pipes
2732                                          ((i * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + Npipes[0] + Npipes[1]) * 4};
2733             const PetscInt pipe_hi[3] = {// vertex numbers of pipes
2734                                          (((i + 1) * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j + 1) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + 1 + Npipes[0] + Npipes[1]) * 4};
2735             for (PetscInt dir = 0; dir < 3; dir++) { // x,y,z
2736               const PetscInt ijk[3] = {i, j, k};
2737               for (PetscInt l = 0; l < 4; l++) { // rotations
2738                 cells[J][dir * 2 + 0][l][0] = pipe_lo[dir] + l;
2739                 cells[J][dir * 2 + 0][l][1] = Jvoff + jfaces[dir][0][l];
2740                 cells[J][dir * 2 + 0][l][2] = Jvoff + jfaces[dir][0][(l - 1 + 4) % 4];
2741                 cells[J][dir * 2 + 0][l][3] = pipe_lo[dir] + (l - 1 + 4) % 4;
2742                 cells[J][dir * 2 + 1][l][0] = Jvoff + jfaces[dir][1][l];
2743                 cells[J][dir * 2 + 1][l][1] = pipe_hi[dir] + l;
2744                 cells[J][dir * 2 + 1][l][2] = pipe_hi[dir] + (l - 1 + 4) % 4;
2745                 cells[J][dir * 2 + 1][l][3] = Jvoff + jfaces[dir][1][(l - 1 + 4) % 4];
2746                 if (ijk[dir] == 0) {
2747                   edges[numEdges][0] = pipe_lo[dir] + l;
2748                   edges[numEdges][1] = pipe_lo[dir] + (l + 1) % 4;
2749                   edgeSets[numEdges] = dir * 2 + 1;
2750                   numEdges++;
2751                 }
2752                 if (ijk[dir] + 1 == extent[dir]) {
2753                   edges[numEdges][0] = pipe_hi[dir] + l;
2754                   edges[numEdges][1] = pipe_hi[dir] + (l + 1) % 4;
2755                   edgeSets[numEdges] = dir * 2 + 2;
2756                   numEdges++;
2757                 }
2758               }
2759             }
2760           }
2761         }
2762       }
2763       PetscCheck(numEdges == Ncuts * 4, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Edge count %" PetscInt_FMT " incompatible with number of cuts %" PetscInt_FMT, numEdges, Ncuts);
2764       numFaces   = 24 * Njunctions;
2765       cells_flat = cells[0][0][0];
2766     }
2767     evalFunc   = TPSEvaluate_SchwarzP;
2768     normalFunc = TPSExtrudeNormalFunc_SchwarzP;
2769     break;
2770   case DMPLEX_TPS_GYROID:
2771     if (rank == 0) {
2772       // This is a coarse mesh approximation of the gyroid shifted to being the zero of the level set
2773       //
2774       //     sin(pi*x)*cos(pi*(y+1/2)) + sin(pi*(y+1/2))*cos(pi*(z+1/4)) + sin(pi*(z+1/4))*cos(x)
2775       //
2776       // on the cell [0,2]^3.
2777       //
2778       // Think about dividing that cell into four columns, and focus on the column [0,1]x[0,1]x[0,2].
2779       // If you looked at the gyroid in that column at different slices of z you would see that it kind of spins
2780       // like a boomerang:
2781       //
2782       //     z = 0          z = 1/4        z = 1/2        z = 3/4     //
2783       //     -----          -------        -------        -------     //
2784       //                                                              //
2785       //     +       +      +       +      +       +      +   \   +   //
2786       //      \                                   /            \      //
2787       //       \            `-_   _-'            /              }     //
2788       //        *-_            `-'            _-'              /      //
2789       //     +     `-+      +       +      +-'     +      +   /   +   //
2790       //                                                              //
2791       //                                                              //
2792       //     z = 1          z = 5/4        z = 3/2        z = 7/4     //
2793       //     -----          -------        -------        -------     //
2794       //                                                              //
2795       //     +-_     +      +       +      +     _-+      +   /   +   //
2796       //        `-_            _-_            _-`            /        //
2797       //           \        _-'   `-_        /              {         //
2798       //            \                       /                \        //
2799       //     +       +      +       +      +       +      +   \   +   //
2800       //
2801       //
2802       // This course mesh approximates each of these slices by two line segments,
2803       // and then connects the segments in consecutive layers with quadrilateral faces.
2804       // All of the end points of the segments are multiples of 1/4 except for the
2805       // point * in the picture for z = 0 above and the similar points in other layers.
2806       // That point is at (gamma, gamma, 0), where gamma is calculated below.
2807       //
2808       // The column  [1,2]x[1,2]x[0,2] looks the same as this column;
2809       // The columns [1,2]x[0,1]x[0,2] and [0,1]x[1,2]x[0,2] are mirror images.
2810       //
2811       // As for how this method turned into the names given to the vertices:
2812       // that was not systematic, it was just the way it worked out in my handwritten notes.
2813 
2814       PetscInt       facesPerBlock = 64;
2815       PetscInt       vertsPerBlock = 56;
2816       PetscInt       extentPlus[3];
2817       PetscInt       numBlocks, numBlocksPlus;
2818       const PetscInt A = 0, B = 1, C = 2, D = 3, E = 4, F = 5, G = 6, H = 7, II = 8, J = 9, K = 10, L = 11, M = 12, N = 13, O = 14, P = 15, Q = 16, R = 17, S = 18, T = 19, U = 20, V = 21, W = 22, X = 23, Y = 24, Z = 25, Ap = 26, Bp = 27, Cp = 28, Dp = 29, Ep = 30, Fp = 31, Gp = 32, Hp = 33, Ip = 34, Jp = 35, Kp = 36, Lp = 37, Mp = 38, Np = 39, Op = 40, Pp = 41, Qp = 42, Rp = 43, Sp = 44, Tp = 45, Up = 46, Vp = 47, Wp = 48, Xp = 49, Yp = 50, Zp = 51, Aq = 52, Bq = 53, Cq = 54, Dq = 55;
2819       const PetscInt pattern[64][4] = {
2820   /* face to vertex within the coarse discretization of a single gyroid block */
2821   /* layer 0 */
2822         {A,           C,           K,           G          },
2823         {C,           B,           II,          K          },
2824         {D,           A,           H,           L          },
2825         {B + 56 * 1,  D,           L,           J          },
2826         {E,           B + 56 * 1,  J,           N          },
2827         {A + 56 * 2,  E,           N,           H + 56 * 2 },
2828         {F,           A + 56 * 2,  G + 56 * 2,  M          },
2829         {B,           F,           M,           II         },
2830  /* layer 1 */
2831         {G,           K,           Q,           O          },
2832         {K,           II,          P,           Q          },
2833         {L,           H,           O + 56 * 1,  R          },
2834         {J,           L,           R,           P          },
2835         {N,           J,           P,           S          },
2836         {H + 56 * 2,  N,           S,           O + 56 * 3 },
2837         {M,           G + 56 * 2,  O + 56 * 2,  T          },
2838         {II,          M,           T,           P          },
2839  /* layer 2 */
2840         {O,           Q,           Y,           U          },
2841         {Q,           P,           W,           Y          },
2842         {R,           O + 56 * 1,  U + 56 * 1,  Ap         },
2843         {P,           R,           Ap,          W          },
2844         {S,           P,           X,           Bp         },
2845         {O + 56 * 3,  S,           Bp,          V + 56 * 1 },
2846         {T,           O + 56 * 2,  V,           Z          },
2847         {P,           T,           Z,           X          },
2848  /* layer 3 */
2849         {U,           Y,           Ep,          Dp         },
2850         {Y,           W,           Cp,          Ep         },
2851         {Ap,          U + 56 * 1,  Dp + 56 * 1, Gp         },
2852         {W,           Ap,          Gp,          Cp         },
2853         {Bp,          X,           Cp + 56 * 2, Fp         },
2854         {V + 56 * 1,  Bp,          Fp,          Dp + 56 * 1},
2855         {Z,           V,           Dp,          Hp         },
2856         {X,           Z,           Hp,          Cp + 56 * 2},
2857  /* layer 4 */
2858         {Dp,          Ep,          Mp,          Kp         },
2859         {Ep,          Cp,          Ip,          Mp         },
2860         {Gp,          Dp + 56 * 1, Lp,          Np         },
2861         {Cp,          Gp,          Np,          Jp         },
2862         {Fp,          Cp + 56 * 2, Jp + 56 * 2, Pp         },
2863         {Dp + 56 * 1, Fp,          Pp,          Lp         },
2864         {Hp,          Dp,          Kp,          Op         },
2865         {Cp + 56 * 2, Hp,          Op,          Ip + 56 * 2},
2866  /* layer 5 */
2867         {Kp,          Mp,          Sp,          Rp         },
2868         {Mp,          Ip,          Qp,          Sp         },
2869         {Np,          Lp,          Rp,          Tp         },
2870         {Jp,          Np,          Tp,          Qp + 56 * 1},
2871         {Pp,          Jp + 56 * 2, Qp + 56 * 3, Up         },
2872         {Lp,          Pp,          Up,          Rp         },
2873         {Op,          Kp,          Rp,          Vp         },
2874         {Ip + 56 * 2, Op,          Vp,          Qp + 56 * 2},
2875  /* layer 6 */
2876         {Rp,          Sp,          Aq,          Yp         },
2877         {Sp,          Qp,          Wp,          Aq         },
2878         {Tp,          Rp,          Yp,          Cq         },
2879         {Qp + 56 * 1, Tp,          Cq,          Wp + 56 * 1},
2880         {Up,          Qp + 56 * 3, Xp + 56 * 1, Dq         },
2881         {Rp,          Up,          Dq,          Zp         },
2882         {Vp,          Rp,          Zp,          Bq         },
2883         {Qp + 56 * 2, Vp,          Bq,          Xp         },
2884  /* layer 7 (the top is the periodic image of the bottom of layer 0) */
2885         {Yp,          Aq,          C + 56 * 4,  A + 56 * 4 },
2886         {Aq,          Wp,          B + 56 * 4,  C + 56 * 4 },
2887         {Cq,          Yp,          A + 56 * 4,  D + 56 * 4 },
2888         {Wp + 56 * 1, Cq,          D + 56 * 4,  B + 56 * 5 },
2889         {Dq,          Xp + 56 * 1, B + 56 * 5,  E + 56 * 4 },
2890         {Zp,          Dq,          E + 56 * 4,  A + 56 * 6 },
2891         {Bq,          Zp,          A + 56 * 6,  F + 56 * 4 },
2892         {Xp,          Bq,          F + 56 * 4,  B + 56 * 4 }
2893       };
2894       const PetscReal gamma                = PetscAcosReal((PetscSqrtReal(3.) - 1.) / PetscSqrtReal(2.)) / PETSC_PI;
2895       const PetscReal patternCoords[56][3] = {
2896         {1.,        0.,        0.  }, /* A  */
2897         {0.,        1.,        0.  }, /* B  */
2898         {gamma,     gamma,     0.  }, /* C  */
2899         {1 + gamma, 1 - gamma, 0.  }, /* D  */
2900         {2 - gamma, 2 - gamma, 0.  }, /* E  */
2901         {1 - gamma, 1 + gamma, 0.  }, /* F  */
2902 
2903         {.5,        0,         .25 }, /* G  */
2904         {1.5,       0.,        .25 }, /* H  */
2905         {.5,        1.,        .25 }, /* II */
2906         {1.5,       1.,        .25 }, /* J  */
2907         {.25,       .5,        .25 }, /* K  */
2908         {1.25,      .5,        .25 }, /* L  */
2909         {.75,       1.5,       .25 }, /* M  */
2910         {1.75,      1.5,       .25 }, /* N  */
2911 
2912         {0.,        0.,        .5  }, /* O  */
2913         {1.,        1.,        .5  }, /* P  */
2914         {gamma,     1 - gamma, .5  }, /* Q  */
2915         {1 + gamma, gamma,     .5  }, /* R  */
2916         {2 - gamma, 1 + gamma, .5  }, /* S  */
2917         {1 - gamma, 2 - gamma, .5  }, /* T  */
2918 
2919         {0.,        .5,        .75 }, /* U  */
2920         {0.,        1.5,       .75 }, /* V  */
2921         {1.,        .5,        .75 }, /* W  */
2922         {1.,        1.5,       .75 }, /* X  */
2923         {.5,        .75,       .75 }, /* Y  */
2924         {.5,        1.75,      .75 }, /* Z  */
2925         {1.5,       .25,       .75 }, /* Ap */
2926         {1.5,       1.25,      .75 }, /* Bp */
2927 
2928         {1.,        0.,        1.  }, /* Cp */
2929         {0.,        1.,        1.  }, /* Dp */
2930         {1 - gamma, 1 - gamma, 1.  }, /* Ep */
2931         {1 + gamma, 1 + gamma, 1.  }, /* Fp */
2932         {2 - gamma, gamma,     1.  }, /* Gp */
2933         {gamma,     2 - gamma, 1.  }, /* Hp */
2934 
2935         {.5,        0.,        1.25}, /* Ip */
2936         {1.5,       0.,        1.25}, /* Jp */
2937         {.5,        1.,        1.25}, /* Kp */
2938         {1.5,       1.,        1.25}, /* Lp */
2939         {.75,       .5,        1.25}, /* Mp */
2940         {1.75,      .5,        1.25}, /* Np */
2941         {.25,       1.5,       1.25}, /* Op */
2942         {1.25,      1.5,       1.25}, /* Pp */
2943 
2944         {0.,        0.,        1.5 }, /* Qp */
2945         {1.,        1.,        1.5 }, /* Rp */
2946         {1 - gamma, gamma,     1.5 }, /* Sp */
2947         {2 - gamma, 1 - gamma, 1.5 }, /* Tp */
2948         {1 + gamma, 2 - gamma, 1.5 }, /* Up */
2949         {gamma,     1 + gamma, 1.5 }, /* Vp */
2950 
2951         {0.,        .5,        1.75}, /* Wp */
2952         {0.,        1.5,       1.75}, /* Xp */
2953         {1.,        .5,        1.75}, /* Yp */
2954         {1.,        1.5,       1.75}, /* Zp */
2955         {.5,        .25,       1.75}, /* Aq */
2956         {.5,        1.25,      1.75}, /* Bq */
2957         {1.5,       .75,       1.75}, /* Cq */
2958         {1.5,       1.75,      1.75}, /* Dq */
2959       };
2960       PetscInt(*cells)[64][4] = NULL;
2961       PetscBool *seen;
2962       PetscInt  *vertToTrueVert;
2963       PetscInt   count;
2964 
2965       for (PetscInt i = 0; i < 3; i++) extentPlus[i] = extent[i] + 1;
2966       numBlocks = 1;
2967       for (PetscInt i = 0; i < 3; i++) numBlocks *= extent[i];
2968       numBlocksPlus = 1;
2969       for (PetscInt i = 0; i < 3; i++) numBlocksPlus *= extentPlus[i];
2970       numFaces = numBlocks * facesPerBlock;
2971       PetscCall(PetscMalloc1(numBlocks, &cells));
2972       PetscCall(PetscCalloc1(numBlocksPlus * vertsPerBlock, &seen));
2973       for (PetscInt k = 0; k < extent[2]; k++) {
2974         for (PetscInt j = 0; j < extent[1]; j++) {
2975           for (PetscInt i = 0; i < extent[0]; i++) {
2976             for (PetscInt f = 0; f < facesPerBlock; f++) {
2977               for (PetscInt v = 0; v < 4; v++) {
2978                 PetscInt vertRaw     = pattern[f][v];
2979                 PetscInt blockidx    = vertRaw / 56;
2980                 PetscInt patternvert = vertRaw % 56;
2981                 PetscInt xplus       = (blockidx & 1);
2982                 PetscInt yplus       = (blockidx & 2) >> 1;
2983                 PetscInt zplus       = (blockidx & 4) >> 2;
2984                 PetscInt zcoord      = (periodic && periodic[2] == DM_BOUNDARY_PERIODIC) ? ((k + zplus) % extent[2]) : (k + zplus);
2985                 PetscInt ycoord      = (periodic && periodic[1] == DM_BOUNDARY_PERIODIC) ? ((j + yplus) % extent[1]) : (j + yplus);
2986                 PetscInt xcoord      = (periodic && periodic[0] == DM_BOUNDARY_PERIODIC) ? ((i + xplus) % extent[0]) : (i + xplus);
2987                 PetscInt vert        = ((zcoord * extentPlus[1] + ycoord) * extentPlus[0] + xcoord) * 56 + patternvert;
2988 
2989                 cells[(k * extent[1] + j) * extent[0] + i][f][v] = vert;
2990                 seen[vert]                                       = PETSC_TRUE;
2991               }
2992             }
2993           }
2994         }
2995       }
2996       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++)
2997         if (seen[i]) numVertices++;
2998       count = 0;
2999       PetscCall(PetscMalloc1(numBlocksPlus * vertsPerBlock, &vertToTrueVert));
3000       PetscCall(PetscMalloc1(numVertices * 3, &vtxCoords));
3001       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) vertToTrueVert[i] = -1;
3002       for (PetscInt k = 0; k < extentPlus[2]; k++) {
3003         for (PetscInt j = 0; j < extentPlus[1]; j++) {
3004           for (PetscInt i = 0; i < extentPlus[0]; i++) {
3005             for (PetscInt v = 0; v < vertsPerBlock; v++) {
3006               PetscInt vIdx = ((k * extentPlus[1] + j) * extentPlus[0] + i) * vertsPerBlock + v;
3007 
3008               if (seen[vIdx]) {
3009                 PetscInt thisVert;
3010 
3011                 vertToTrueVert[vIdx] = thisVert = count++;
3012 
3013                 for (PetscInt d = 0; d < 3; d++) vtxCoords[3 * thisVert + d] = patternCoords[v][d];
3014                 vtxCoords[3 * thisVert + 0] += i * 2;
3015                 vtxCoords[3 * thisVert + 1] += j * 2;
3016                 vtxCoords[3 * thisVert + 2] += k * 2;
3017               }
3018             }
3019           }
3020         }
3021       }
3022       for (PetscInt i = 0; i < numBlocks; i++) {
3023         for (PetscInt f = 0; f < facesPerBlock; f++) {
3024           for (PetscInt v = 0; v < 4; v++) cells[i][f][v] = vertToTrueVert[cells[i][f][v]];
3025         }
3026       }
3027       PetscCall(PetscFree(vertToTrueVert));
3028       PetscCall(PetscFree(seen));
3029       cells_flat = cells[0][0];
3030       numEdges   = 0;
3031       for (PetscInt i = 0; i < numFaces; i++) {
3032         for (PetscInt e = 0; e < 4; e++) {
3033           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3034           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3035 
3036           for (PetscInt d = 0; d < 3; d++) {
3037             if (!periodic || periodic[0] != DM_BOUNDARY_PERIODIC) {
3038               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) numEdges++;
3039               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) numEdges++;
3040             }
3041           }
3042         }
3043       }
3044       PetscCall(PetscMalloc1(numEdges, &edges));
3045       PetscCall(PetscMalloc1(numEdges, &edgeSets));
3046       for (PetscInt edge = 0, i = 0; i < numFaces; i++) {
3047         for (PetscInt e = 0; e < 4; e++) {
3048           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3049           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3050 
3051           for (PetscInt d = 0; d < 3; d++) {
3052             if (!periodic || periodic[d] != DM_BOUNDARY_PERIODIC) {
3053               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) {
3054                 edges[edge][0]   = ev[0];
3055                 edges[edge][1]   = ev[1];
3056                 edgeSets[edge++] = 2 * d;
3057               }
3058               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) {
3059                 edges[edge][0]   = ev[0];
3060                 edges[edge][1]   = ev[1];
3061                 edgeSets[edge++] = 2 * d + 1;
3062               }
3063             }
3064           }
3065         }
3066       }
3067     }
3068     evalFunc   = TPSEvaluate_Gyroid;
3069     normalFunc = TPSExtrudeNormalFunc_Gyroid;
3070     break;
3071   }
3072 
3073   PetscCall(DMSetDimension(dm, topoDim));
3074   if (rank == 0) PetscCall(DMPlexBuildFromCellList(dm, numFaces, numVertices, 4, cells_flat));
3075   else PetscCall(DMPlexBuildFromCellList(dm, 0, 0, 0, NULL));
3076   PetscCall(PetscFree(cells_flat));
3077   {
3078     DM idm;
3079     PetscCall(DMPlexInterpolate(dm, &idm));
3080     PetscCall(DMPlexReplace_Internal(dm, &idm));
3081   }
3082   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, vtxCoords));
3083   else PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, NULL));
3084   PetscCall(PetscFree(vtxCoords));
3085 
3086   PetscCall(DMCreateLabel(dm, "Face Sets"));
3087   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3088   for (PetscInt e = 0; e < numEdges; e++) {
3089     PetscInt        njoin;
3090     const PetscInt *join, verts[] = {numFaces + edges[e][0], numFaces + edges[e][1]};
3091     PetscCall(DMPlexGetJoin(dm, 2, verts, &njoin, &join));
3092     PetscCheck(njoin == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected unique join of vertices %" PetscInt_FMT " and %" PetscInt_FMT, edges[e][0], edges[e][1]);
3093     PetscCall(DMLabelSetValue(label, join[0], edgeSets[e]));
3094     PetscCall(DMPlexRestoreJoin(dm, 2, verts, &njoin, &join));
3095   }
3096   PetscCall(PetscFree(edges));
3097   PetscCall(PetscFree(edgeSets));
3098   if (tps_distribute) {
3099     DM               pdm = NULL;
3100     PetscPartitioner part;
3101 
3102     PetscCall(DMPlexGetPartitioner(dm, &part));
3103     PetscCall(PetscPartitionerSetFromOptions(part));
3104     PetscCall(DMPlexDistribute(dm, 0, NULL, &pdm));
3105     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
3106     // Do not auto-distribute again
3107     PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
3108   }
3109 
3110   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3111   for (PetscInt refine = 0; refine < refinements; refine++) {
3112     PetscInt     m;
3113     DM           dmf;
3114     Vec          X;
3115     PetscScalar *x;
3116     PetscCall(DMRefine(dm, MPI_COMM_NULL, &dmf));
3117     PetscCall(DMPlexReplace_Internal(dm, &dmf));
3118 
3119     PetscCall(DMGetCoordinatesLocal(dm, &X));
3120     PetscCall(VecGetLocalSize(X, &m));
3121     PetscCall(VecGetArray(X, &x));
3122     for (PetscInt i = 0; i < m; i += 3) PetscCall(TPSNearestPoint(evalFunc, &x[i]));
3123     PetscCall(VecRestoreArray(X, &x));
3124   }
3125 
3126   // Face Sets has already been propagated to new vertices during refinement; this propagates to the initial vertices.
3127   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3128   PetscCall(DMPlexLabelComplete(dm, label));
3129 
3130   if (thickness > 0) {
3131     DM              edm, cdm, ecdm;
3132     DMPlexTransform tr;
3133     const char     *prefix;
3134     PetscOptions    options;
3135     // Code from DMPlexExtrude
3136     PetscCall(DMPlexTransformCreate(PetscObjectComm((PetscObject)dm), &tr));
3137     PetscCall(DMPlexTransformSetDM(tr, dm));
3138     PetscCall(DMPlexTransformSetType(tr, DMPLEXEXTRUDE));
3139     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
3140     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)tr, prefix));
3141     PetscCall(PetscObjectGetOptions((PetscObject)dm, &options));
3142     PetscCall(PetscObjectSetOptions((PetscObject)tr, options));
3143     PetscCall(DMPlexTransformExtrudeSetLayers(tr, layers));
3144     PetscCall(DMPlexTransformExtrudeSetThickness(tr, thickness));
3145     PetscCall(DMPlexTransformExtrudeSetTensor(tr, PETSC_FALSE));
3146     PetscCall(DMPlexTransformExtrudeSetSymmetric(tr, PETSC_TRUE));
3147     PetscCall(DMPlexTransformExtrudeSetNormalFunction(tr, normalFunc));
3148     PetscCall(DMPlexTransformSetFromOptions(tr));
3149     PetscCall(PetscObjectSetOptions((PetscObject)tr, NULL));
3150     PetscCall(DMPlexTransformSetUp(tr));
3151     PetscCall(PetscObjectViewFromOptions((PetscObject)tr, NULL, "-dm_plex_tps_transform_view"));
3152     PetscCall(DMPlexTransformApply(tr, dm, &edm));
3153     PetscCall(DMCopyDisc(dm, edm));
3154     PetscCall(DMGetCoordinateDM(dm, &cdm));
3155     PetscCall(DMGetCoordinateDM(edm, &ecdm));
3156     PetscCall(DMCopyDisc(cdm, ecdm));
3157     PetscCall(DMPlexTransformCreateDiscLabels(tr, edm));
3158     PetscCall(DMPlexTransformDestroy(&tr));
3159     if (edm) {
3160       ((DM_Plex *)edm->data)->printFEM    = ((DM_Plex *)dm->data)->printFEM;
3161       ((DM_Plex *)edm->data)->printL2     = ((DM_Plex *)dm->data)->printL2;
3162       ((DM_Plex *)edm->data)->printLocate = ((DM_Plex *)dm->data)->printLocate;
3163     }
3164     PetscCall(DMPlexReplace_Internal(dm, &edm));
3165   }
3166   PetscFunctionReturn(0);
3167 }
3168 
3169 /*@
3170   DMPlexCreateTPSMesh - Create a distributed, interpolated mesh of a triply-periodic surface
3171 
3172   Collective
3173 
3174   Input Parameters:
3175 + comm   - The communicator for the DM object
3176 . tpstype - Type of triply-periodic surface
3177 . extent - Array of length 3 containing number of periods in each direction
3178 . periodic - array of length 3 with periodicity, or NULL for non-periodic
3179 . tps_distribute - Distribute 2D manifold mesh prior to refinement and extrusion (more scalable)
3180 . refinements - Number of factor-of-2 refinements of 2D manifold mesh
3181 . layers - Number of cell layers extruded in normal direction
3182 - thickness - Thickness in normal direction
3183 
3184   Output Parameter:
3185 . dm  - The DM object
3186 
3187   Notes:
3188   This meshes the surface of the Schwarz P or Gyroid surfaces.  Schwarz P is is the simplest member of the triply-periodic minimal surfaces.
3189   https://en.wikipedia.org/wiki/Schwarz_minimal_surface#Schwarz_P_(%22Primitive%22) and can be cut with "clean" boundaries.
3190   The Gyroid (https://en.wikipedia.org/wiki/Gyroid) is another triply-periodic minimal surface with applications in additive manufacturing; it is much more difficult to "cut" since there are no planes of symmetry.
3191   Our implementation creates a very coarse mesh of the surface and refines (by 4-way splitting) as many times as requested.
3192   On each refinement, all vertices are projected to their nearest point on the surface.
3193   This projection could readily be extended to related surfaces.
3194 
3195   The face (edge) sets for the Schwarz P surface are numbered 1(-x), 2(+x), 3(-y), 4(+y), 5(-z), 6(+z).
3196   When the mesh is refined, "Face Sets" contain the new vertices (created during refinement).  Use DMPlexLabelComplete() to propagate to coarse-level vertices.
3197 
3198   References:
3199 . * - Maskery et al, Insights into the mechanical properties of several triply periodic minimal surface lattice structures made by polymer additive manufacturing, 2017. https://doi.org/10.1016/j.polymer.2017.11.049
3200 
3201   Developer Notes:
3202   The Gyroid mesh does not currently mark boundary sets.
3203 
3204   Level: beginner
3205 
3206 .seealso: `DMPlexCreateSphereMesh()`, `DMSetType()`, `DMCreate()`
3207 @*/
3208 PetscErrorCode DMPlexCreateTPSMesh(MPI_Comm comm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness, DM *dm) {
3209   PetscFunctionBegin;
3210   PetscCall(DMCreate(comm, dm));
3211   PetscCall(DMSetType(*dm, DMPLEX));
3212   PetscCall(DMPlexCreateTPSMesh_Internal(*dm, tpstype, extent, periodic, tps_distribute, refinements, layers, thickness));
3213   PetscFunctionReturn(0);
3214 }
3215 
3216 /*@
3217   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
3218 
3219   Collective
3220 
3221   Input Parameters:
3222 + comm    - The communicator for the DM object
3223 . dim     - The dimension
3224 . simplex - Use simplices, or tensor product cells
3225 - R       - The radius
3226 
3227   Output Parameter:
3228 . dm  - The DM object
3229 
3230   Level: beginner
3231 
3232 .seealso: `DMPlexCreateBallMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3233 @*/
3234 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscReal R, DM *dm) {
3235   PetscFunctionBegin;
3236   PetscValidPointer(dm, 5);
3237   PetscCall(DMCreate(comm, dm));
3238   PetscCall(DMSetType(*dm, DMPLEX));
3239   PetscCall(DMPlexCreateSphereMesh_Internal(*dm, dim, simplex, R));
3240   PetscFunctionReturn(0);
3241 }
3242 
3243 static PetscErrorCode DMPlexCreateBallMesh_Internal(DM dm, PetscInt dim, PetscReal R) {
3244   DM      sdm, vol;
3245   DMLabel bdlabel;
3246 
3247   PetscFunctionBegin;
3248   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
3249   PetscCall(DMSetType(sdm, DMPLEX));
3250   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sdm, "bd_"));
3251   PetscCall(DMPlexCreateSphereMesh_Internal(sdm, dim - 1, PETSC_TRUE, R));
3252   PetscCall(DMSetFromOptions(sdm));
3253   PetscCall(DMViewFromOptions(sdm, NULL, "-dm_view"));
3254   PetscCall(DMPlexGenerate(sdm, NULL, PETSC_TRUE, &vol));
3255   PetscCall(DMDestroy(&sdm));
3256   PetscCall(DMPlexReplace_Internal(dm, &vol));
3257   PetscCall(DMCreateLabel(dm, "marker"));
3258   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
3259   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
3260   PetscCall(DMPlexLabelComplete(dm, bdlabel));
3261   PetscFunctionReturn(0);
3262 }
3263 
3264 /*@
3265   DMPlexCreateBallMesh - Creates a simplex mesh on the d-dimensional ball, B^d.
3266 
3267   Collective
3268 
3269   Input Parameters:
3270 + comm  - The communicator for the DM object
3271 . dim   - The dimension
3272 - R     - The radius
3273 
3274   Output Parameter:
3275 . dm  - The DM object
3276 
3277   Options Database Keys:
3278 - bd_dm_refine - This will refine the surface mesh preserving the sphere geometry
3279 
3280   Level: beginner
3281 
3282 .seealso: `DMPlexCreateSphereMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3283 @*/
3284 PetscErrorCode DMPlexCreateBallMesh(MPI_Comm comm, PetscInt dim, PetscReal R, DM *dm) {
3285   PetscFunctionBegin;
3286   PetscCall(DMCreate(comm, dm));
3287   PetscCall(DMSetType(*dm, DMPLEX));
3288   PetscCall(DMPlexCreateBallMesh_Internal(*dm, dim, R));
3289   PetscFunctionReturn(0);
3290 }
3291 
3292 static PetscErrorCode DMPlexCreateReferenceCell_Internal(DM rdm, DMPolytopeType ct) {
3293   PetscFunctionBegin;
3294   switch (ct) {
3295   case DM_POLYTOPE_POINT: {
3296     PetscInt    numPoints[1]        = {1};
3297     PetscInt    coneSize[1]         = {0};
3298     PetscInt    cones[1]            = {0};
3299     PetscInt    coneOrientations[1] = {0};
3300     PetscScalar vertexCoords[1]     = {0.0};
3301 
3302     PetscCall(DMSetDimension(rdm, 0));
3303     PetscCall(DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3304   } break;
3305   case DM_POLYTOPE_SEGMENT: {
3306     PetscInt    numPoints[2]        = {2, 1};
3307     PetscInt    coneSize[3]         = {2, 0, 0};
3308     PetscInt    cones[2]            = {1, 2};
3309     PetscInt    coneOrientations[2] = {0, 0};
3310     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3311 
3312     PetscCall(DMSetDimension(rdm, 1));
3313     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3314   } break;
3315   case DM_POLYTOPE_POINT_PRISM_TENSOR: {
3316     PetscInt    numPoints[2]        = {2, 1};
3317     PetscInt    coneSize[3]         = {2, 0, 0};
3318     PetscInt    cones[2]            = {1, 2};
3319     PetscInt    coneOrientations[2] = {0, 0};
3320     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3321 
3322     PetscCall(DMSetDimension(rdm, 1));
3323     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3324   } break;
3325   case DM_POLYTOPE_TRIANGLE: {
3326     PetscInt    numPoints[2]        = {3, 1};
3327     PetscInt    coneSize[4]         = {3, 0, 0, 0};
3328     PetscInt    cones[3]            = {1, 2, 3};
3329     PetscInt    coneOrientations[3] = {0, 0, 0};
3330     PetscScalar vertexCoords[6]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0};
3331 
3332     PetscCall(DMSetDimension(rdm, 2));
3333     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3334   } break;
3335   case DM_POLYTOPE_QUADRILATERAL: {
3336     PetscInt    numPoints[2]        = {4, 1};
3337     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3338     PetscInt    cones[4]            = {1, 2, 3, 4};
3339     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3340     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0};
3341 
3342     PetscCall(DMSetDimension(rdm, 2));
3343     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3344   } break;
3345   case DM_POLYTOPE_SEG_PRISM_TENSOR: {
3346     PetscInt    numPoints[2]        = {4, 1};
3347     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3348     PetscInt    cones[4]            = {1, 2, 3, 4};
3349     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3350     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0};
3351 
3352     PetscCall(DMSetDimension(rdm, 2));
3353     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3354   } break;
3355   case DM_POLYTOPE_TETRAHEDRON: {
3356     PetscInt    numPoints[2]        = {4, 1};
3357     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3358     PetscInt    cones[4]            = {1, 2, 3, 4};
3359     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3360     PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0};
3361 
3362     PetscCall(DMSetDimension(rdm, 3));
3363     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3364   } break;
3365   case DM_POLYTOPE_HEXAHEDRON: {
3366     PetscInt    numPoints[2]        = {8, 1};
3367     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3368     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3369     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3370     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3371 
3372     PetscCall(DMSetDimension(rdm, 3));
3373     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3374   } break;
3375   case DM_POLYTOPE_TRI_PRISM: {
3376     PetscInt    numPoints[2]        = {6, 1};
3377     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3378     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3379     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3380     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3381 
3382     PetscCall(DMSetDimension(rdm, 3));
3383     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3384   } break;
3385   case DM_POLYTOPE_TRI_PRISM_TENSOR: {
3386     PetscInt    numPoints[2]        = {6, 1};
3387     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3388     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3389     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3390     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3391 
3392     PetscCall(DMSetDimension(rdm, 3));
3393     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3394   } break;
3395   case DM_POLYTOPE_QUAD_PRISM_TENSOR: {
3396     PetscInt    numPoints[2]        = {8, 1};
3397     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3398     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3399     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3400     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3401 
3402     PetscCall(DMSetDimension(rdm, 3));
3403     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3404   } break;
3405   case DM_POLYTOPE_PYRAMID: {
3406     PetscInt    numPoints[2]        = {5, 1};
3407     PetscInt    coneSize[6]         = {5, 0, 0, 0, 0, 0};
3408     PetscInt    cones[5]            = {1, 2, 3, 4, 5};
3409     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3410     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, 0.0, 0.0, 1.0};
3411 
3412     PetscCall(DMSetDimension(rdm, 3));
3413     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3414   } break;
3415   default: SETERRQ(PetscObjectComm((PetscObject)rdm), PETSC_ERR_ARG_WRONG, "Cannot create reference cell for cell type %s", DMPolytopeTypes[ct]);
3416   }
3417   {
3418     PetscInt Nv, v;
3419 
3420     /* Must create the celltype label here so that we do not automatically try to compute the types */
3421     PetscCall(DMCreateLabel(rdm, "celltype"));
3422     PetscCall(DMPlexSetCellType(rdm, 0, ct));
3423     PetscCall(DMPlexGetChart(rdm, NULL, &Nv));
3424     for (v = 1; v < Nv; ++v) PetscCall(DMPlexSetCellType(rdm, v, DM_POLYTOPE_POINT));
3425   }
3426   PetscCall(DMPlexInterpolateInPlace_Internal(rdm));
3427   PetscCall(PetscObjectSetName((PetscObject)rdm, DMPolytopeTypes[ct]));
3428   PetscFunctionReturn(0);
3429 }
3430 
3431 /*@
3432   DMPlexCreateReferenceCell - Create a DMPLEX with the appropriate FEM reference cell
3433 
3434   Collective
3435 
3436   Input Parameters:
3437 + comm - The communicator
3438 - ct   - The cell type of the reference cell
3439 
3440   Output Parameter:
3441 . refdm - The reference cell
3442 
3443   Level: intermediate
3444 
3445 .seealso: `DMPlexCreateReferenceCell()`, `DMPlexCreateBoxMesh()`
3446 @*/
3447 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, DMPolytopeType ct, DM *refdm) {
3448   PetscFunctionBegin;
3449   PetscCall(DMCreate(comm, refdm));
3450   PetscCall(DMSetType(*refdm, DMPLEX));
3451   PetscCall(DMPlexCreateReferenceCell_Internal(*refdm, ct));
3452   PetscFunctionReturn(0);
3453 }
3454 
3455 static PetscErrorCode DMPlexCreateBoundaryLabel_Private(DM dm, const char name[]) {
3456   DM        plex;
3457   DMLabel   label;
3458   PetscBool hasLabel;
3459 
3460   PetscFunctionBegin;
3461   PetscCall(DMHasLabel(dm, name, &hasLabel));
3462   if (hasLabel) PetscFunctionReturn(0);
3463   PetscCall(DMCreateLabel(dm, name));
3464   PetscCall(DMGetLabel(dm, name, &label));
3465   PetscCall(DMConvert(dm, DMPLEX, &plex));
3466   PetscCall(DMPlexMarkBoundaryFaces(plex, 1, label));
3467   PetscCall(DMPlexLabelComplete(plex, label));
3468   PetscCall(DMDestroy(&plex));
3469   PetscFunctionReturn(0);
3470 }
3471 
3472 /*
3473   We use the last coordinate as the radius, the inner radius is lower[dim-1] and the outer radius is upper[dim-1]. Then we map the first coordinate around the circle.
3474 
3475     (x, y) -> (r, theta) = (x[1], (x[0] - lower[0]) * 2\pi/(upper[0] - lower[0]))
3476 */
3477 static void boxToAnnulus(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[]) {
3478   const PetscReal low = PetscRealPart(constants[0]);
3479   const PetscReal upp = PetscRealPart(constants[1]);
3480   const PetscReal r   = PetscRealPart(u[1]);
3481   const PetscReal th  = 2. * PETSC_PI * (PetscRealPart(u[0]) - low) / (upp - low);
3482 
3483   f0[0] = r * PetscCosReal(th);
3484   f0[1] = r * PetscSinReal(th);
3485 }
3486 
3487 const char *const DMPlexShapes[] = {"box", "box_surface", "ball", "sphere", "cylinder", "schwarz_p", "gyroid", "doublet", "annulus", "unknown", "DMPlexShape", "DM_SHAPE_", NULL};
3488 
3489 static PetscErrorCode DMPlexCreateFromOptions_Internal(PetscOptionItems *PetscOptionsObject, PetscBool *useCoordSpace, DM dm) {
3490   DMPlexShape    shape   = DM_SHAPE_BOX;
3491   DMPolytopeType cell    = DM_POLYTOPE_TRIANGLE;
3492   PetscInt       dim     = 2;
3493   PetscBool      simplex = PETSC_TRUE, interpolate = PETSC_TRUE, adjCone = PETSC_FALSE, adjClosure = PETSC_TRUE, refDomain = PETSC_FALSE;
3494   PetscBool      flg, flg2, fflg, bdfflg, nameflg;
3495   MPI_Comm       comm;
3496   char           filename[PETSC_MAX_PATH_LEN]   = "<unspecified>";
3497   char           bdFilename[PETSC_MAX_PATH_LEN] = "<unspecified>";
3498   char           plexname[PETSC_MAX_PATH_LEN]   = "";
3499 
3500   PetscFunctionBegin;
3501   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3502   /* TODO Turn this into a registration interface */
3503   PetscCall(PetscOptionsString("-dm_plex_filename", "File containing a mesh", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &fflg));
3504   PetscCall(PetscOptionsString("-dm_plex_boundary_filename", "File containing a mesh boundary", "DMPlexCreateFromFile", bdFilename, bdFilename, sizeof(bdFilename), &bdfflg));
3505   PetscCall(PetscOptionsString("-dm_plex_name", "Name of the mesh in the file", "DMPlexCreateFromFile", plexname, plexname, sizeof(plexname), &nameflg));
3506   PetscCall(PetscOptionsEnum("-dm_plex_cell", "Cell shape", "", DMPolytopeTypes, (PetscEnum)cell, (PetscEnum *)&cell, NULL));
3507   PetscCall(PetscOptionsBool("-dm_plex_reference_cell_domain", "Use a reference cell domain", "", refDomain, &refDomain, NULL));
3508   PetscCall(PetscOptionsEnum("-dm_plex_shape", "Shape for built-in mesh", "", DMPlexShapes, (PetscEnum)shape, (PetscEnum *)&shape, &flg));
3509   PetscCall(PetscOptionsBoundedInt("-dm_plex_dim", "Topological dimension of the mesh", "DMGetDimension", dim, &dim, &flg, 0));
3510   PetscCheck(!(dim < 0) && !(dim > 3), comm, PETSC_ERR_ARG_OUTOFRANGE, "Dimension %" PetscInt_FMT " should be in [1, 3]", dim);
3511   PetscCall(PetscOptionsBool("-dm_plex_simplex", "Mesh cell shape", "", simplex, &simplex, &flg));
3512   PetscCall(PetscOptionsBool("-dm_plex_interpolate", "Flag to create edges and faces automatically", "", interpolate, &interpolate, &flg));
3513   PetscCall(PetscOptionsBool("-dm_plex_adj_cone", "Set adjacency direction", "DMSetBasicAdjacency", adjCone, &adjCone, &flg));
3514   PetscCall(PetscOptionsBool("-dm_plex_adj_closure", "Set adjacency size", "DMSetBasicAdjacency", adjClosure, &adjClosure, &flg2));
3515   if (flg || flg2) PetscCall(DMSetBasicAdjacency(dm, adjCone, adjClosure));
3516 
3517   switch (cell) {
3518   case DM_POLYTOPE_POINT:
3519   case DM_POLYTOPE_SEGMENT:
3520   case DM_POLYTOPE_POINT_PRISM_TENSOR:
3521   case DM_POLYTOPE_TRIANGLE:
3522   case DM_POLYTOPE_QUADRILATERAL:
3523   case DM_POLYTOPE_TETRAHEDRON:
3524   case DM_POLYTOPE_HEXAHEDRON: *useCoordSpace = PETSC_TRUE; break;
3525   default: *useCoordSpace = PETSC_FALSE; break;
3526   }
3527 
3528   if (fflg) {
3529     DM dmnew;
3530 
3531     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), filename, plexname, interpolate, &dmnew));
3532     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3533     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3534   } else if (refDomain) {
3535     PetscCall(DMPlexCreateReferenceCell_Internal(dm, cell));
3536   } else if (bdfflg) {
3537     DM bdm, dmnew;
3538 
3539     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), bdFilename, plexname, interpolate, &bdm));
3540     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)bdm, "bd_"));
3541     PetscCall(DMSetFromOptions(bdm));
3542     PetscCall(DMPlexGenerate(bdm, NULL, interpolate, &dmnew));
3543     PetscCall(DMDestroy(&bdm));
3544     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3545     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3546   } else {
3547     PetscCall(PetscObjectSetName((PetscObject)dm, DMPlexShapes[shape]));
3548     switch (shape) {
3549     case DM_SHAPE_BOX:
3550     case DM_SHAPE_ANNULUS: {
3551       PetscInt       faces[3]  = {0, 0, 0};
3552       PetscReal      lower[3]  = {0, 0, 0};
3553       PetscReal      upper[3]  = {1, 1, 1};
3554       DMBoundaryType bdt[3]    = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3555       PetscBool      isAnnular = shape == DM_SHAPE_ANNULUS ? PETSC_TRUE : PETSC_FALSE;
3556       PetscInt       i, n;
3557 
3558       n = dim;
3559       for (i = 0; i < dim; ++i) faces[i] = (dim == 1 ? 1 : 4 - dim);
3560       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3561       n = 3;
3562       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3563       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3564       n = 3;
3565       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3566       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3567       n = 3;
3568       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
3569       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3570 
3571       PetscCheck(!isAnnular || dim == 2, comm, PETSC_ERR_ARG_OUTOFRANGE, "Only two dimensional annuli have been implemented");
3572       if (isAnnular)
3573         for (i = 0; i < dim - 1; ++i) bdt[i] = DM_BOUNDARY_PERIODIC;
3574 
3575       switch (cell) {
3576       case DM_POLYTOPE_TRI_PRISM_TENSOR:
3577         PetscCall(DMPlexCreateWedgeBoxMesh_Internal(dm, faces, lower, upper, bdt));
3578         if (!interpolate) {
3579           DM udm;
3580 
3581           PetscCall(DMPlexUninterpolate(dm, &udm));
3582           PetscCall(DMPlexReplace_Internal(dm, &udm));
3583         }
3584         break;
3585       default: PetscCall(DMPlexCreateBoxMesh_Internal(dm, dim, simplex, faces, lower, upper, bdt, interpolate)); break;
3586       }
3587       if (isAnnular) {
3588         DM          cdm;
3589         PetscDS     cds;
3590         PetscScalar bounds[2] = {lower[0], upper[0]};
3591 
3592         // Fix coordinates for annular region
3593         PetscCall(DMSetPeriodicity(dm, NULL, NULL, NULL));
3594         PetscCall(DMSetCellCoordinatesLocal(dm, NULL));
3595         PetscCall(DMSetCellCoordinates(dm, NULL));
3596         PetscCall(DMPlexCreateCoordinateSpace(dm, 1, NULL));
3597         PetscCall(DMGetCoordinateDM(dm, &cdm));
3598         PetscCall(DMGetDS(cdm, &cds));
3599         PetscCall(PetscDSSetConstants(cds, 2, bounds));
3600         PetscCall(DMPlexRemapGeometry(dm, 0.0, boxToAnnulus));
3601       }
3602     } break;
3603     case DM_SHAPE_BOX_SURFACE: {
3604       PetscInt  faces[3] = {0, 0, 0};
3605       PetscReal lower[3] = {0, 0, 0};
3606       PetscReal upper[3] = {1, 1, 1};
3607       PetscInt  i, n;
3608 
3609       n = dim + 1;
3610       for (i = 0; i < dim + 1; ++i) faces[i] = (dim + 1 == 1 ? 1 : 4 - (dim + 1));
3611       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3612       n = 3;
3613       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3614       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3615       n = 3;
3616       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3617       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3618       PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(dm, dim + 1, faces, lower, upper, interpolate));
3619     } break;
3620     case DM_SHAPE_SPHERE: {
3621       PetscReal R = 1.0;
3622 
3623       PetscCall(PetscOptionsReal("-dm_plex_sphere_radius", "Radius of the sphere", "", R, &R, &flg));
3624       PetscCall(DMPlexCreateSphereMesh_Internal(dm, dim, simplex, R));
3625     } break;
3626     case DM_SHAPE_BALL: {
3627       PetscReal R = 1.0;
3628 
3629       PetscCall(PetscOptionsReal("-dm_plex_ball_radius", "Radius of the ball", "", R, &R, &flg));
3630       PetscCall(DMPlexCreateBallMesh_Internal(dm, dim, R));
3631     } break;
3632     case DM_SHAPE_CYLINDER: {
3633       DMBoundaryType bdt = DM_BOUNDARY_NONE;
3634       PetscInt       Nw  = 6;
3635 
3636       PetscCall(PetscOptionsEnum("-dm_plex_cylinder_bd", "Boundary type in the z direction", "", DMBoundaryTypes, (PetscEnum)bdt, (PetscEnum *)&bdt, NULL));
3637       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_wedges", "Number of wedges around the cylinder", "", Nw, &Nw, NULL));
3638       switch (cell) {
3639       case DM_POLYTOPE_TRI_PRISM_TENSOR: PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(dm, Nw, interpolate)); break;
3640       default: PetscCall(DMPlexCreateHexCylinderMesh_Internal(dm, bdt)); break;
3641       }
3642     } break;
3643     case DM_SHAPE_SCHWARZ_P: // fallthrough
3644     case DM_SHAPE_GYROID: {
3645       PetscInt       extent[3] = {1, 1, 1}, refine = 0, layers = 0, three;
3646       PetscReal      thickness   = 0.;
3647       DMBoundaryType periodic[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3648       DMPlexTPSType  tps_type    = shape == DM_SHAPE_SCHWARZ_P ? DMPLEX_TPS_SCHWARZ_P : DMPLEX_TPS_GYROID;
3649       PetscBool      tps_distribute;
3650       PetscCall(PetscOptionsIntArray("-dm_plex_tps_extent", "Number of replicas for each of three dimensions", NULL, extent, (three = 3, &three), NULL));
3651       PetscCall(PetscOptionsInt("-dm_plex_tps_refine", "Number of refinements", NULL, refine, &refine, NULL));
3652       PetscCall(PetscOptionsEnumArray("-dm_plex_tps_periodic", "Periodicity in each of three dimensions", NULL, DMBoundaryTypes, (PetscEnum *)periodic, (three = 3, &three), NULL));
3653       PetscCall(PetscOptionsInt("-dm_plex_tps_layers", "Number of layers in volumetric extrusion (or zero to not extrude)", NULL, layers, &layers, NULL));
3654       PetscCall(PetscOptionsReal("-dm_plex_tps_thickness", "Thickness of volumetric extrusion", NULL, thickness, &thickness, NULL));
3655       PetscCall(DMPlexDistributeGetDefault(dm, &tps_distribute));
3656       PetscCall(PetscOptionsBool("-dm_plex_tps_distribute", "Distribute the 2D mesh prior to refinement and extrusion", NULL, tps_distribute, &tps_distribute, NULL));
3657       PetscCall(DMPlexCreateTPSMesh_Internal(dm, tps_type, extent, periodic, tps_distribute, refine, layers, thickness));
3658     } break;
3659     case DM_SHAPE_DOUBLET: {
3660       DM        dmnew;
3661       PetscReal rl = 0.0;
3662 
3663       PetscCall(PetscOptionsReal("-dm_plex_doublet_refinementlimit", "Refinement limit", NULL, rl, &rl, NULL));
3664       PetscCall(DMPlexCreateDoublet(PetscObjectComm((PetscObject)dm), dim, simplex, interpolate, rl, &dmnew));
3665       PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3666       PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3667     } break;
3668     default: SETERRQ(comm, PETSC_ERR_SUP, "Domain shape %s is unsupported", DMPlexShapes[shape]);
3669     }
3670   }
3671   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3672   if (!((PetscObject)dm)->name && nameflg) PetscCall(PetscObjectSetName((PetscObject)dm, plexname));
3673   PetscFunctionReturn(0);
3674 }
3675 
3676 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(DM dm, PetscOptionItems *PetscOptionsObject) {
3677   DM_Plex  *mesh = (DM_Plex *)dm->data;
3678   PetscBool flg, flg2;
3679   char      bdLabel[PETSC_MAX_PATH_LEN];
3680 
3681   PetscFunctionBegin;
3682   /* Handle viewing */
3683   PetscCall(PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL));
3684   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fem", "Debug output level all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL, 0));
3685   PetscCall(PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL));
3686   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL, 0));
3687   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_locate", "Debug output level all point location computations", "DMLocatePoints", 0, &mesh->printLocate, NULL, 0));
3688   PetscCall(DMMonitorSetFromOptions(dm, "-dm_plex_monitor_throughput", "Monitor the simulation throughput", "DMPlexMonitorThroughput", DMPlexMonitorThroughput, NULL, &flg));
3689   if (flg) PetscCall(PetscLogDefaultBegin());
3690   /* Labeling */
3691   PetscCall(PetscOptionsString("-dm_plex_boundary_label", "Label to mark the mesh boundary", "", bdLabel, bdLabel, sizeof(bdLabel), &flg));
3692   if (flg) PetscCall(DMPlexCreateBoundaryLabel_Private(dm, bdLabel));
3693   /* Point Location */
3694   PetscCall(PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL));
3695   /* Partitioning and distribution */
3696   PetscCall(PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL));
3697   /* Generation and remeshing */
3698   PetscCall(PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL));
3699   /* Projection behavior */
3700   PetscCall(PetscOptionsBoundedInt("-dm_plex_max_projection_height", "Maxmimum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL, 0));
3701   PetscCall(PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL));
3702   /* Checking structure */
3703   {
3704     PetscBool all = PETSC_FALSE;
3705 
3706     PetscCall(PetscOptionsBool("-dm_plex_check_all", "Perform all basic checks", "DMPlexCheck", PETSC_FALSE, &all, NULL));
3707     if (all) {
3708       PetscCall(DMPlexCheck(dm));
3709     } else {
3710       PetscCall(PetscOptionsBool("-dm_plex_check_symmetry", "Check that the adjacency information in the mesh is symmetric", "DMPlexCheckSymmetry", PETSC_FALSE, &flg, &flg2));
3711       if (flg && flg2) PetscCall(DMPlexCheckSymmetry(dm));
3712       PetscCall(PetscOptionsBool("-dm_plex_check_skeleton", "Check that each cell has the correct number of vertices (only for homogeneous simplex or tensor meshes)", "DMPlexCheckSkeleton", PETSC_FALSE, &flg, &flg2));
3713       if (flg && flg2) PetscCall(DMPlexCheckSkeleton(dm, 0));
3714       PetscCall(PetscOptionsBool("-dm_plex_check_faces", "Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type", "DMPlexCheckFaces", PETSC_FALSE, &flg, &flg2));
3715       if (flg && flg2) PetscCall(DMPlexCheckFaces(dm, 0));
3716       PetscCall(PetscOptionsBool("-dm_plex_check_geometry", "Check that cells have positive volume", "DMPlexCheckGeometry", PETSC_FALSE, &flg, &flg2));
3717       if (flg && flg2) PetscCall(DMPlexCheckGeometry(dm));
3718       PetscCall(PetscOptionsBool("-dm_plex_check_pointsf", "Check some necessary conditions for PointSF", "DMPlexCheckPointSF", PETSC_FALSE, &flg, &flg2));
3719       if (flg && flg2) PetscCall(DMPlexCheckPointSF(dm, NULL, PETSC_FALSE));
3720       PetscCall(PetscOptionsBool("-dm_plex_check_interface_cones", "Check points on inter-partition interfaces have conforming order of cone points", "DMPlexCheckInterfaceCones", PETSC_FALSE, &flg, &flg2));
3721       if (flg && flg2) PetscCall(DMPlexCheckInterfaceCones(dm));
3722     }
3723     PetscCall(PetscOptionsBool("-dm_plex_check_cell_shape", "Check cell shape", "DMPlexCheckCellShape", PETSC_FALSE, &flg, &flg2));
3724     if (flg && flg2) PetscCall(DMPlexCheckCellShape(dm, PETSC_TRUE, PETSC_DETERMINE));
3725   }
3726   {
3727     PetscReal scale = 1.0;
3728 
3729     PetscCall(PetscOptionsReal("-dm_plex_scale", "Scale factor for mesh coordinates", "DMPlexScale", scale, &scale, &flg));
3730     if (flg) {
3731       Vec coordinates, coordinatesLocal;
3732 
3733       PetscCall(DMGetCoordinates(dm, &coordinates));
3734       PetscCall(DMGetCoordinatesLocal(dm, &coordinatesLocal));
3735       PetscCall(VecScale(coordinates, scale));
3736       PetscCall(VecScale(coordinatesLocal, scale));
3737     }
3738   }
3739   PetscCall(PetscPartitionerSetFromOptions(mesh->partitioner));
3740   PetscFunctionReturn(0);
3741 }
3742 
3743 PetscErrorCode DMSetFromOptions_Overlap_Plex(DM dm, PetscOptionItems *PetscOptionsObject, PetscInt *overlap) {
3744   PetscInt  numOvLabels = 16, numOvExLabels = 16;
3745   char     *ovLabelNames[16], *ovExLabelNames[16];
3746   PetscInt  numOvValues = 16, numOvExValues = 16, l;
3747   PetscBool flg;
3748 
3749   PetscFunctionBegin;
3750   PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", *overlap, overlap, NULL, 0));
3751   PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_labels", "List of overlap label names", "DMPlexDistribute", ovLabelNames, &numOvLabels, &flg));
3752   if (!flg) numOvLabels = 0;
3753   if (numOvLabels) {
3754     ((DM_Plex *)dm->data)->numOvLabels = numOvLabels;
3755     for (l = 0; l < numOvLabels; ++l) {
3756       PetscCall(DMGetLabel(dm, ovLabelNames[l], &((DM_Plex *)dm->data)->ovLabels[l]));
3757       PetscCheck(((DM_Plex *)dm->data)->ovLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovLabelNames[l]);
3758       PetscCall(PetscFree(ovLabelNames[l]));
3759     }
3760     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_values", "List of overlap label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovValues, &numOvValues, &flg));
3761     if (!flg) numOvValues = 0;
3762     PetscCheck(numOvLabels == numOvValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvLabels, numOvValues);
3763 
3764     PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_exclude_labels", "List of overlap exclude label names", "DMPlexDistribute", ovExLabelNames, &numOvExLabels, &flg));
3765     if (!flg) numOvExLabels = 0;
3766     ((DM_Plex *)dm->data)->numOvExLabels = numOvExLabels;
3767     for (l = 0; l < numOvExLabels; ++l) {
3768       PetscCall(DMGetLabel(dm, ovExLabelNames[l], &((DM_Plex *)dm->data)->ovExLabels[l]));
3769       PetscCheck(((DM_Plex *)dm->data)->ovExLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovExLabelNames[l]);
3770       PetscCall(PetscFree(ovExLabelNames[l]));
3771     }
3772     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_exclude_values", "List of overlap exclude label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovExValues, &numOvExValues, &flg));
3773     if (!flg) numOvExValues = 0;
3774     PetscCheck(numOvExLabels == numOvExValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of exclude labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvExLabels, numOvExValues);
3775   }
3776   PetscFunctionReturn(0);
3777 }
3778 
3779 static PetscErrorCode DMSetFromOptions_Plex(DM dm, PetscOptionItems *PetscOptionsObject) {
3780   PetscFunctionList        ordlist;
3781   char                     oname[256];
3782   PetscReal                volume    = -1.0;
3783   PetscInt                 prerefine = 0, refine = 0, r, coarsen = 0, overlap = 0, extLayers = 0, dim;
3784   PetscBool                uniformOrig, created = PETSC_FALSE, uniform = PETSC_TRUE, distribute, interpolate = PETSC_TRUE, coordSpace = PETSC_TRUE, remap = PETSC_TRUE, ghostCells = PETSC_FALSE, isHierarchy, ignoreModel = PETSC_FALSE, flg;
3785   DMPlexReorderDefaultFlag reorder;
3786 
3787   PetscFunctionBegin;
3788   PetscOptionsHeadBegin(PetscOptionsObject, "DMPlex Options");
3789   /* Handle automatic creation */
3790   PetscCall(DMGetDimension(dm, &dim));
3791   if (dim < 0) {
3792     PetscCall(DMPlexCreateFromOptions_Internal(PetscOptionsObject, &coordSpace, dm));
3793     created = PETSC_TRUE;
3794   }
3795   PetscCall(DMGetDimension(dm, &dim));
3796   /* Handle interpolation before distribution */
3797   PetscCall(PetscOptionsBool("-dm_plex_interpolate_pre", "Flag to interpolate mesh before distribution", "", interpolate, &interpolate, &flg));
3798   if (flg) {
3799     DMPlexInterpolatedFlag interpolated;
3800 
3801     PetscCall(DMPlexIsInterpolated(dm, &interpolated));
3802     if (interpolated == DMPLEX_INTERPOLATED_FULL && !interpolate) {
3803       DM udm;
3804 
3805       PetscCall(DMPlexUninterpolate(dm, &udm));
3806       PetscCall(DMPlexReplace_Internal(dm, &udm));
3807     } else if (interpolated != DMPLEX_INTERPOLATED_FULL && interpolate) {
3808       DM idm;
3809 
3810       PetscCall(DMPlexInterpolate(dm, &idm));
3811       PetscCall(DMPlexReplace_Internal(dm, &idm));
3812     }
3813   }
3814   /* Handle DMPlex refinement before distribution */
3815   PetscCall(PetscOptionsBool("-dm_refine_ignore_model", "Flag to ignore the geometry model when refining", "DMCreate", ignoreModel, &ignoreModel, &flg));
3816   if (flg) ((DM_Plex *)dm->data)->ignoreModel = ignoreModel;
3817   PetscCall(DMPlexGetRefinementUniform(dm, &uniformOrig));
3818   PetscCall(PetscOptionsBoundedInt("-dm_refine_pre", "The number of refinements before distribution", "DMCreate", prerefine, &prerefine, NULL, 0));
3819   PetscCall(PetscOptionsBool("-dm_refine_remap_pre", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
3820   PetscCall(PetscOptionsBool("-dm_refine_uniform_pre", "Flag for uniform refinement before distribution", "DMCreate", uniform, &uniform, &flg));
3821   if (flg) PetscCall(DMPlexSetRefinementUniform(dm, uniform));
3822   PetscCall(PetscOptionsReal("-dm_refine_volume_limit_pre", "The maximum cell volume after refinement before distribution", "DMCreate", volume, &volume, &flg));
3823   if (flg) {
3824     PetscCall(DMPlexSetRefinementUniform(dm, PETSC_FALSE));
3825     PetscCall(DMPlexSetRefinementLimit(dm, volume));
3826     prerefine = PetscMax(prerefine, 1);
3827   }
3828   for (r = 0; r < prerefine; ++r) {
3829     DM             rdm;
3830     PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
3831 
3832     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3833     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
3834     PetscCall(DMPlexReplace_Internal(dm, &rdm));
3835     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3836     if (coordFunc && remap) {
3837       PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
3838       ((DM_Plex *)dm->data)->coordFunc = coordFunc;
3839     }
3840   }
3841   PetscCall(DMPlexSetRefinementUniform(dm, uniformOrig));
3842   /* Handle DMPlex extrusion before distribution */
3843   PetscCall(PetscOptionsBoundedInt("-dm_extrude", "The number of layers to extrude", "", extLayers, &extLayers, NULL, 0));
3844   if (extLayers) {
3845     DM edm;
3846 
3847     PetscCall(DMExtrude(dm, extLayers, &edm));
3848     PetscCall(DMPlexReplace_Internal(dm, &edm));
3849     ((DM_Plex *)dm->data)->coordFunc = NULL;
3850     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3851     extLayers = 0;
3852   }
3853   /* Handle DMPlex reordering before distribution */
3854   PetscCall(DMPlexReorderGetDefault(dm, &reorder));
3855   PetscCall(MatGetOrderingList(&ordlist));
3856   PetscCall(PetscStrncpy(oname, MATORDERINGNATURAL, sizeof(oname)));
3857   PetscCall(PetscOptionsFList("-dm_plex_reorder", "Set mesh reordering type", "DMPlexGetOrdering", ordlist, MATORDERINGNATURAL, oname, sizeof(oname), &flg));
3858   if (reorder == DMPLEX_REORDER_DEFAULT_TRUE || flg) {
3859     DM pdm;
3860     IS perm;
3861 
3862     PetscCall(DMPlexGetOrdering(dm, oname, NULL, &perm));
3863     PetscCall(DMPlexPermute(dm, perm, &pdm));
3864     PetscCall(ISDestroy(&perm));
3865     PetscCall(DMPlexReplace_Internal(dm, &pdm));
3866     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3867   }
3868   /* Handle DMPlex distribution */
3869   PetscCall(DMPlexDistributeGetDefault(dm, &distribute));
3870   PetscCall(PetscOptionsBool("-dm_distribute", "Flag to redistribute a mesh among processes", "DMPlexDistribute", distribute, &distribute, NULL));
3871   PetscCall(DMSetFromOptions_Overlap_Plex(dm, PetscOptionsObject, &overlap));
3872   if (distribute) {
3873     DM               pdm = NULL;
3874     PetscPartitioner part;
3875 
3876     PetscCall(DMPlexGetPartitioner(dm, &part));
3877     PetscCall(PetscPartitionerSetFromOptions(part));
3878     PetscCall(DMPlexDistribute(dm, overlap, NULL, &pdm));
3879     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
3880   }
3881   /* Create coordinate space */
3882   if (created) {
3883     DM_Plex  *mesh   = (DM_Plex *)dm->data;
3884     PetscInt  degree = 1;
3885     PetscBool flg;
3886 
3887     PetscCall(PetscOptionsBool("-dm_coord_space", "Use an FEM space for coordinates", "", coordSpace, &coordSpace, &flg));
3888     PetscCall(PetscOptionsInt("-dm_coord_petscspace_degree", "FEM degree for coordinate space", "", degree, &degree, NULL));
3889     if (coordSpace) PetscCall(DMPlexCreateCoordinateSpace(dm, degree, mesh->coordFunc));
3890     if (flg && !coordSpace) {
3891       DM           cdm;
3892       PetscDS      cds;
3893       PetscObject  obj;
3894       PetscClassId id;
3895 
3896       PetscCall(DMGetCoordinateDM(dm, &cdm));
3897       PetscCall(DMGetDS(cdm, &cds));
3898       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
3899       PetscCall(PetscObjectGetClassId(obj, &id));
3900       if (id == PETSCFE_CLASSID) {
3901         PetscContainer dummy;
3902 
3903         PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &dummy));
3904         PetscCall(PetscObjectSetName((PetscObject)dummy, "coordinates"));
3905         PetscCall(DMSetField(cdm, 0, NULL, (PetscObject)dummy));
3906         PetscCall(PetscContainerDestroy(&dummy));
3907         PetscCall(DMClearDS(cdm));
3908       }
3909       mesh->coordFunc = NULL;
3910     }
3911     PetscCall(PetscOptionsBool("-dm_sparse_localize", "Localize only necessary cells", "", dm->sparseLocalize, &dm->sparseLocalize, &flg));
3912     PetscCall(DMLocalizeCoordinates(dm));
3913   }
3914   /* Handle DMPlex refinement */
3915   remap = PETSC_TRUE;
3916   PetscCall(PetscOptionsBoundedInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL, 0));
3917   PetscCall(PetscOptionsBool("-dm_refine_remap", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
3918   PetscCall(PetscOptionsBoundedInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy, 0));
3919   if (refine) PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3920   if (refine && isHierarchy) {
3921     DM *dms, coarseDM;
3922 
3923     PetscCall(DMGetCoarseDM(dm, &coarseDM));
3924     PetscCall(PetscObjectReference((PetscObject)coarseDM));
3925     PetscCall(PetscMalloc1(refine, &dms));
3926     PetscCall(DMRefineHierarchy(dm, refine, dms));
3927     /* Total hack since we do not pass in a pointer */
3928     PetscCall(DMPlexSwap_Static(dm, dms[refine - 1]));
3929     if (refine == 1) {
3930       PetscCall(DMSetCoarseDM(dm, dms[0]));
3931       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
3932     } else {
3933       PetscCall(DMSetCoarseDM(dm, dms[refine - 2]));
3934       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
3935       PetscCall(DMSetCoarseDM(dms[0], dms[refine - 1]));
3936       PetscCall(DMPlexSetRegularRefinement(dms[0], PETSC_TRUE));
3937     }
3938     PetscCall(DMSetCoarseDM(dms[refine - 1], coarseDM));
3939     PetscCall(PetscObjectDereference((PetscObject)coarseDM));
3940     /* Free DMs */
3941     for (r = 0; r < refine; ++r) {
3942       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
3943       PetscCall(DMDestroy(&dms[r]));
3944     }
3945     PetscCall(PetscFree(dms));
3946   } else {
3947     for (r = 0; r < refine; ++r) {
3948       DM             rdm;
3949       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
3950 
3951       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3952       PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
3953       /* Total hack since we do not pass in a pointer */
3954       PetscCall(DMPlexReplace_Internal(dm, &rdm));
3955       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3956       if (coordFunc && remap) {
3957         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
3958         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
3959       }
3960     }
3961   }
3962   /* Handle DMPlex coarsening */
3963   PetscCall(PetscOptionsBoundedInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL, 0));
3964   PetscCall(PetscOptionsBoundedInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy, 0));
3965   if (coarsen && isHierarchy) {
3966     DM *dms;
3967 
3968     PetscCall(PetscMalloc1(coarsen, &dms));
3969     PetscCall(DMCoarsenHierarchy(dm, coarsen, dms));
3970     /* Free DMs */
3971     for (r = 0; r < coarsen; ++r) {
3972       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
3973       PetscCall(DMDestroy(&dms[r]));
3974     }
3975     PetscCall(PetscFree(dms));
3976   } else {
3977     for (r = 0; r < coarsen; ++r) {
3978       DM             cdm;
3979       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
3980 
3981       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3982       PetscCall(DMCoarsen(dm, PetscObjectComm((PetscObject)dm), &cdm));
3983       /* Total hack since we do not pass in a pointer */
3984       PetscCall(DMPlexReplace_Internal(dm, &cdm));
3985       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3986       if (coordFunc) {
3987         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
3988         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
3989       }
3990     }
3991   }
3992   /* Handle ghost cells */
3993   PetscCall(PetscOptionsBool("-dm_plex_create_fv_ghost_cells", "Flag to create finite volume ghost cells on the boundary", "DMCreate", ghostCells, &ghostCells, NULL));
3994   if (ghostCells) {
3995     DM   gdm;
3996     char lname[PETSC_MAX_PATH_LEN];
3997 
3998     lname[0] = '\0';
3999     PetscCall(PetscOptionsString("-dm_plex_fv_ghost_cells_label", "Label name for ghost cells boundary", "DMCreate", lname, lname, sizeof(lname), &flg));
4000     PetscCall(DMPlexConstructGhostCells(dm, flg ? lname : NULL, NULL, &gdm));
4001     PetscCall(DMPlexReplace_Internal(dm, &gdm));
4002   }
4003   /* Handle 1D order */
4004   if (reorder != DMPLEX_REORDER_DEFAULT_FALSE && dim == 1) {
4005     DM           cdm, rdm;
4006     PetscDS      cds;
4007     PetscObject  obj;
4008     PetscClassId id = PETSC_OBJECT_CLASSID;
4009     IS           perm;
4010     PetscInt     Nf;
4011     PetscBool    distributed;
4012 
4013     PetscCall(DMPlexIsDistributed(dm, &distributed));
4014     PetscCall(DMGetCoordinateDM(dm, &cdm));
4015     PetscCall(DMGetDS(cdm, &cds));
4016     PetscCall(PetscDSGetNumFields(cds, &Nf));
4017     if (Nf) {
4018       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
4019       PetscCall(PetscObjectGetClassId(obj, &id));
4020     }
4021     if (!distributed && id != PETSCFE_CLASSID) {
4022       PetscCall(DMPlexGetOrdering1D(dm, &perm));
4023       PetscCall(DMPlexPermute(dm, perm, &rdm));
4024       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4025       PetscCall(ISDestroy(&perm));
4026     }
4027   }
4028   /* Handle */
4029   PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4030   PetscOptionsHeadEnd();
4031   PetscFunctionReturn(0);
4032 }
4033 
4034 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm, Vec *vec) {
4035   PetscFunctionBegin;
4036   PetscCall(DMCreateGlobalVector_Section_Private(dm, vec));
4037   /* PetscCall(VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM)); */
4038   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex));
4039   PetscCall(VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void))VecView_Plex_Native));
4040   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex));
4041   PetscCall(VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void))VecLoad_Plex_Native));
4042   PetscFunctionReturn(0);
4043 }
4044 
4045 static PetscErrorCode DMCreateLocalVector_Plex(DM dm, Vec *vec) {
4046   PetscFunctionBegin;
4047   PetscCall(DMCreateLocalVector_Section_Private(dm, vec));
4048   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex_Local));
4049   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex_Local));
4050   PetscFunctionReturn(0);
4051 }
4052 
4053 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd) {
4054   PetscInt depth, d;
4055 
4056   PetscFunctionBegin;
4057   PetscCall(DMPlexGetDepth(dm, &depth));
4058   if (depth == 1) {
4059     PetscCall(DMGetDimension(dm, &d));
4060     if (dim == 0) PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4061     else if (dim == d) PetscCall(DMPlexGetDepthStratum(dm, 1, pStart, pEnd));
4062     else {
4063       *pStart = 0;
4064       *pEnd   = 0;
4065     }
4066   } else {
4067     PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4068   }
4069   PetscFunctionReturn(0);
4070 }
4071 
4072 static PetscErrorCode DMGetNeighbors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[]) {
4073   PetscSF            sf;
4074   PetscInt           niranks, njranks, n;
4075   const PetscMPIInt *iranks, *jranks;
4076   DM_Plex           *data = (DM_Plex *)dm->data;
4077 
4078   PetscFunctionBegin;
4079   PetscCall(DMGetPointSF(dm, &sf));
4080   if (!data->neighbors) {
4081     PetscCall(PetscSFSetUp(sf));
4082     PetscCall(PetscSFGetRootRanks(sf, &njranks, &jranks, NULL, NULL, NULL));
4083     PetscCall(PetscSFGetLeafRanks(sf, &niranks, &iranks, NULL, NULL));
4084     PetscCall(PetscMalloc1(njranks + niranks + 1, &data->neighbors));
4085     PetscCall(PetscArraycpy(data->neighbors + 1, jranks, njranks));
4086     PetscCall(PetscArraycpy(data->neighbors + njranks + 1, iranks, niranks));
4087     n = njranks + niranks;
4088     PetscCall(PetscSortRemoveDupsMPIInt(&n, data->neighbors + 1));
4089     /* The following cast should never fail: can't have more neighbors than PETSC_MPI_INT_MAX */
4090     PetscCall(PetscMPIIntCast(n, data->neighbors));
4091   }
4092   if (nranks) *nranks = data->neighbors[0];
4093   if (ranks) {
4094     if (data->neighbors[0]) *ranks = data->neighbors + 1;
4095     else *ranks = NULL;
4096   }
4097   PetscFunctionReturn(0);
4098 }
4099 
4100 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM, DM, Mat, Vec, Vec);
4101 
4102 static PetscErrorCode DMInitialize_Plex(DM dm) {
4103   PetscFunctionBegin;
4104   dm->ops->view                      = DMView_Plex;
4105   dm->ops->load                      = DMLoad_Plex;
4106   dm->ops->setfromoptions            = DMSetFromOptions_Plex;
4107   dm->ops->clone                     = DMClone_Plex;
4108   dm->ops->setup                     = DMSetUp_Plex;
4109   dm->ops->createlocalsection        = DMCreateLocalSection_Plex;
4110   dm->ops->createdefaultconstraints  = DMCreateDefaultConstraints_Plex;
4111   dm->ops->createglobalvector        = DMCreateGlobalVector_Plex;
4112   dm->ops->createlocalvector         = DMCreateLocalVector_Plex;
4113   dm->ops->getlocaltoglobalmapping   = NULL;
4114   dm->ops->createfieldis             = NULL;
4115   dm->ops->createcoordinatedm        = DMCreateCoordinateDM_Plex;
4116   dm->ops->createcoordinatefield     = DMCreateCoordinateField_Plex;
4117   dm->ops->getcoloring               = NULL;
4118   dm->ops->creatematrix              = DMCreateMatrix_Plex;
4119   dm->ops->createinterpolation       = DMCreateInterpolation_Plex;
4120   dm->ops->createmassmatrix          = DMCreateMassMatrix_Plex;
4121   dm->ops->createmassmatrixlumped    = DMCreateMassMatrixLumped_Plex;
4122   dm->ops->createinjection           = DMCreateInjection_Plex;
4123   dm->ops->refine                    = DMRefine_Plex;
4124   dm->ops->coarsen                   = DMCoarsen_Plex;
4125   dm->ops->refinehierarchy           = DMRefineHierarchy_Plex;
4126   dm->ops->coarsenhierarchy          = DMCoarsenHierarchy_Plex;
4127   dm->ops->extrude                   = DMExtrude_Plex;
4128   dm->ops->globaltolocalbegin        = NULL;
4129   dm->ops->globaltolocalend          = NULL;
4130   dm->ops->localtoglobalbegin        = NULL;
4131   dm->ops->localtoglobalend          = NULL;
4132   dm->ops->destroy                   = DMDestroy_Plex;
4133   dm->ops->createsubdm               = DMCreateSubDM_Plex;
4134   dm->ops->createsuperdm             = DMCreateSuperDM_Plex;
4135   dm->ops->getdimpoints              = DMGetDimPoints_Plex;
4136   dm->ops->locatepoints              = DMLocatePoints_Plex;
4137   dm->ops->projectfunctionlocal      = DMProjectFunctionLocal_Plex;
4138   dm->ops->projectfunctionlabellocal = DMProjectFunctionLabelLocal_Plex;
4139   dm->ops->projectfieldlocal         = DMProjectFieldLocal_Plex;
4140   dm->ops->projectfieldlabellocal    = DMProjectFieldLabelLocal_Plex;
4141   dm->ops->projectbdfieldlabellocal  = DMProjectBdFieldLabelLocal_Plex;
4142   dm->ops->computel2diff             = DMComputeL2Diff_Plex;
4143   dm->ops->computel2gradientdiff     = DMComputeL2GradientDiff_Plex;
4144   dm->ops->computel2fielddiff        = DMComputeL2FieldDiff_Plex;
4145   dm->ops->getneighbors              = DMGetNeighbors_Plex;
4146   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertBoundaryValues_C", DMPlexInsertBoundaryValues_Plex));
4147   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertTimeDerviativeBoundaryValues_C", DMPlexInsertTimeDerivativeBoundaryValues_Plex));
4148   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMSetUpGLVisViewer_C", DMSetUpGLVisViewer_Plex));
4149   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMCreateNeumannOverlap_C", DMCreateNeumannOverlap_Plex));
4150   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4151   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeGetDefault_C", DMPlexDistributeGetDefault_Plex));
4152   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeSetDefault_C", DMPlexDistributeSetDefault_Plex));
4153   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderGetDefault_C", DMPlexReorderGetDefault_Plex));
4154   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderSetDefault_C", DMPlexReorderSetDefault_Plex));
4155   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMInterpolateSolution_C", DMInterpolateSolution_Plex));
4156   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4157   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetOverlap_C", DMPlexSetOverlap_Plex));
4158   PetscFunctionReturn(0);
4159 }
4160 
4161 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm) {
4162   DM_Plex *mesh = (DM_Plex *)dm->data;
4163 
4164   PetscFunctionBegin;
4165   mesh->refct++;
4166   (*newdm)->data = mesh;
4167   PetscCall(PetscObjectChangeTypeName((PetscObject)*newdm, DMPLEX));
4168   PetscCall(DMInitialize_Plex(*newdm));
4169   PetscFunctionReturn(0);
4170 }
4171 
4172 /*MC
4173   DMPLEX = "plex" - A DM object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
4174                     In the local representation, Vecs contain all unknowns in the interior and shared boundary. This is
4175                     specified by a PetscSection object. Ownership in the global representation is determined by
4176                     ownership of the underlying DMPlex points. This is specified by another PetscSection object.
4177 
4178   Options Database Keys:
4179 + -dm_refine_pre                     - Refine mesh before distribution
4180 + -dm_refine_uniform_pre             - Choose uniform or generator-based refinement
4181 + -dm_refine_volume_limit_pre        - Cell volume limit after pre-refinement using generator
4182 . -dm_distribute                     - Distribute mesh across processes
4183 . -dm_distribute_overlap             - Number of cells to overlap for distribution
4184 . -dm_refine                         - Refine mesh after distribution
4185 . -dm_plex_hash_location             - Use grid hashing for point location
4186 . -dm_plex_hash_box_faces <n,m,p>    - The number of divisions in each direction of the grid hash
4187 . -dm_plex_partition_balance         - Attempt to evenly divide points on partition boundary between processes
4188 . -dm_plex_remesh_bd                 - Allow changes to the boundary on remeshing
4189 . -dm_plex_max_projection_height     - Maxmimum mesh point height used to project locally
4190 . -dm_plex_regular_refinement        - Use special nested projection algorithm for regular refinement
4191 . -dm_plex_check_all                 - Perform all shecks below
4192 . -dm_plex_check_symmetry            - Check that the adjacency information in the mesh is symmetric
4193 . -dm_plex_check_skeleton <celltype> - Check that each cell has the correct number of vertices
4194 . -dm_plex_check_faces <celltype>    - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
4195 . -dm_plex_check_geometry            - Check that cells have positive volume
4196 . -dm_view :mesh.tex:ascii_latex     - View the mesh in LaTeX/TikZ
4197 . -dm_plex_view_scale <num>          - Scale the TikZ
4198 - -dm_plex_print_fem <num>           - View FEM assembly information, such as element vectors and matrices
4199 
4200   Level: intermediate
4201 
4202 .seealso: `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`
4203 M*/
4204 
4205 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm) {
4206   DM_Plex *mesh;
4207   PetscInt unit;
4208 
4209   PetscFunctionBegin;
4210   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4211   PetscCall(PetscNew(&mesh));
4212   dm->data = mesh;
4213 
4214   mesh->refct = 1;
4215   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection));
4216   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection));
4217   mesh->refinementUniform      = PETSC_TRUE;
4218   mesh->refinementLimit        = -1.0;
4219   mesh->distDefault            = PETSC_TRUE;
4220   mesh->reorderDefault         = DMPLEX_REORDER_DEFAULT_NOTSET;
4221   mesh->distributionName       = NULL;
4222   mesh->interpolated           = DMPLEX_INTERPOLATED_INVALID;
4223   mesh->interpolatedCollective = DMPLEX_INTERPOLATED_INVALID;
4224 
4225   PetscCall(PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner));
4226   mesh->remeshBd = PETSC_FALSE;
4227 
4228   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
4229 
4230   mesh->depthState    = -1;
4231   mesh->celltypeState = -1;
4232   mesh->printTol      = 1.0e-10;
4233 
4234   PetscCall(DMInitialize_Plex(dm));
4235   PetscFunctionReturn(0);
4236 }
4237 
4238 /*@
4239   DMPlexCreate - Creates a DMPlex object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
4240 
4241   Collective
4242 
4243   Input Parameter:
4244 . comm - The communicator for the DMPlex object
4245 
4246   Output Parameter:
4247 . mesh  - The DMPlex object
4248 
4249   Level: beginner
4250 
4251 @*/
4252 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh) {
4253   PetscFunctionBegin;
4254   PetscValidPointer(mesh, 2);
4255   PetscCall(DMCreate(comm, mesh));
4256   PetscCall(DMSetType(*mesh, DMPLEX));
4257   PetscFunctionReturn(0);
4258 }
4259 
4260 /*@C
4261   DMPlexBuildFromCellListParallel - Build distributed DMPLEX topology from a list of vertices for each cell (common mesh generator output)
4262 
4263   Input Parameters:
4264 + dm - The DM
4265 . numCells - The number of cells owned by this process
4266 . numVertices - The number of vertices to be owned by this process, or PETSC_DECIDE
4267 . NVertices - The global number of vertices, or PETSC_DETERMINE
4268 . numCorners - The number of vertices for each cell
4269 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4270 
4271   Output Parameters:
4272 + vertexSF - (Optional) SF describing complete vertex ownership
4273 - verticesAdjSaved - (Optional) vertex adjacency array
4274 
4275   Notes:
4276   Two triangles sharing a face
4277 $
4278 $        2
4279 $      / | \
4280 $     /  |  \
4281 $    /   |   \
4282 $   0  0 | 1  3
4283 $    \   |   /
4284 $     \  |  /
4285 $      \ | /
4286 $        1
4287 would have input
4288 $  numCells = 2, numVertices = 4
4289 $  cells = [0 1 2  1 3 2]
4290 $
4291 which would result in the DMPlex
4292 $
4293 $        4
4294 $      / | \
4295 $     /  |  \
4296 $    /   |   \
4297 $   2  0 | 1  5
4298 $    \   |   /
4299 $     \  |  /
4300 $      \ | /
4301 $        3
4302 
4303   Vertices are implicitly numbered consecutively 0,...,NVertices.
4304   Each rank owns a chunk of numVertices consecutive vertices.
4305   If numVertices is PETSC_DECIDE, PETSc will distribute them as evenly as possible using PetscLayout.
4306   If NVertices is PETSC_DETERMINE and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
4307   If only NVertices is PETSC_DETERMINE, it is computed as the sum of numVertices over all ranks.
4308 
4309   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
4310 
4311   Not currently supported in Fortran.
4312 
4313   Level: advanced
4314 
4315 .seealso: `DMPlexBuildFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildCoordinatesFromCellListParallel()`
4316 @*/
4317 PetscErrorCode DMPlexBuildFromCellListParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, const PetscInt cells[], PetscSF *vertexSF, PetscInt **verticesAdjSaved) {
4318   PetscSF     sfPoint;
4319   PetscLayout layout;
4320   PetscInt    numVerticesAdj, *verticesAdj, *cones, c, p;
4321 
4322   PetscFunctionBegin;
4323   PetscValidLogicalCollectiveInt(dm, NVertices, 4);
4324   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4325   /* Get/check global number of vertices */
4326   {
4327     PetscInt       NVerticesInCells, i;
4328     const PetscInt len = numCells * numCorners;
4329 
4330     /* NVerticesInCells = max(cells) + 1 */
4331     NVerticesInCells = PETSC_MIN_INT;
4332     for (i = 0; i < len; i++)
4333       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4334     ++NVerticesInCells;
4335     PetscCallMPI(MPI_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
4336 
4337     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
4338     else
4339       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
4340   }
4341   /* Count locally unique vertices */
4342   {
4343     PetscHSetI vhash;
4344     PetscInt   off = 0;
4345 
4346     PetscCall(PetscHSetICreate(&vhash));
4347     for (c = 0; c < numCells; ++c) {
4348       for (p = 0; p < numCorners; ++p) PetscCall(PetscHSetIAdd(vhash, cells[c * numCorners + p]));
4349     }
4350     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
4351     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
4352     else verticesAdj = *verticesAdjSaved;
4353     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
4354     PetscCall(PetscHSetIDestroy(&vhash));
4355     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
4356   }
4357   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
4358   /* Create cones */
4359   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
4360   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
4361   PetscCall(DMSetUp(dm));
4362   PetscCall(DMPlexGetCones(dm, &cones));
4363   for (c = 0; c < numCells; ++c) {
4364     for (p = 0; p < numCorners; ++p) {
4365       const PetscInt gv = cells[c * numCorners + p];
4366       PetscInt       lv;
4367 
4368       /* Positions within verticesAdj form 0-based local vertex numbering;
4369          we need to shift it by numCells to get correct DAG points (cells go first) */
4370       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
4371       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
4372       cones[c * numCorners + p] = lv + numCells;
4373     }
4374   }
4375   /* Build point sf */
4376   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
4377   PetscCall(PetscLayoutSetSize(layout, NVertices));
4378   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
4379   PetscCall(PetscLayoutSetBlockSize(layout, 1));
4380   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
4381   PetscCall(PetscLayoutDestroy(&layout));
4382   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
4383   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
4384   if (dm->sf) {
4385     const char *prefix;
4386 
4387     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
4388     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
4389   }
4390   PetscCall(DMSetPointSF(dm, sfPoint));
4391   PetscCall(PetscSFDestroy(&sfPoint));
4392   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)(*vertexSF), "Vertex Ownership SF"));
4393   /* Fill in the rest of the topology structure */
4394   PetscCall(DMPlexSymmetrize(dm));
4395   PetscCall(DMPlexStratify(dm));
4396   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4397   PetscFunctionReturn(0);
4398 }
4399 
4400 /*@C
4401   DMPlexBuildCoordinatesFromCellListParallel - Build DM coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4402 
4403   Input Parameters:
4404 + dm - The DM
4405 . spaceDim - The spatial dimension used for coordinates
4406 . sfVert - SF describing complete vertex ownership
4407 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4408 
4409   Level: advanced
4410 
4411   Notes:
4412   Not currently supported in Fortran.
4413 
4414 .seealso: `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellListParallel()`
4415 @*/
4416 PetscErrorCode DMPlexBuildCoordinatesFromCellListParallel(DM dm, PetscInt spaceDim, PetscSF sfVert, const PetscReal vertexCoords[]) {
4417   PetscSection coordSection;
4418   Vec          coordinates;
4419   PetscScalar *coords;
4420   PetscInt     numVertices, numVerticesAdj, coordSize, v, vStart, vEnd;
4421 
4422   PetscFunctionBegin;
4423   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4424   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
4425   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4426   PetscCall(DMSetCoordinateDim(dm, spaceDim));
4427   PetscCall(PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL));
4428   PetscCheck(vEnd - vStart == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Supplied sfVert has wrong number of leaves = %" PetscInt_FMT " != %" PetscInt_FMT " = vEnd - vStart", numVerticesAdj, vEnd - vStart);
4429   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4430   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4431   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
4432   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
4433   for (v = vStart; v < vEnd; ++v) {
4434     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
4435     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
4436   }
4437   PetscCall(PetscSectionSetUp(coordSection));
4438   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4439   PetscCall(VecCreate(PetscObjectComm((PetscObject)dm), &coordinates));
4440   PetscCall(VecSetBlockSize(coordinates, spaceDim));
4441   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4442   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4443   PetscCall(VecSetType(coordinates, VECSTANDARD));
4444   PetscCall(VecGetArray(coordinates, &coords));
4445   {
4446     MPI_Datatype coordtype;
4447 
4448     /* Need a temp buffer for coords if we have complex/single */
4449     PetscCallMPI(MPI_Type_contiguous(spaceDim, MPIU_SCALAR, &coordtype));
4450     PetscCallMPI(MPI_Type_commit(&coordtype));
4451 #if defined(PETSC_USE_COMPLEX)
4452     {
4453       PetscScalar *svertexCoords;
4454       PetscInt     i;
4455       PetscCall(PetscMalloc1(numVertices * spaceDim, &svertexCoords));
4456       for (i = 0; i < numVertices * spaceDim; i++) svertexCoords[i] = vertexCoords[i];
4457       PetscCall(PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4458       PetscCall(PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4459       PetscCall(PetscFree(svertexCoords));
4460     }
4461 #else
4462     PetscCall(PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4463     PetscCall(PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4464 #endif
4465     PetscCallMPI(MPI_Type_free(&coordtype));
4466   }
4467   PetscCall(VecRestoreArray(coordinates, &coords));
4468   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4469   PetscCall(VecDestroy(&coordinates));
4470   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4471   PetscFunctionReturn(0);
4472 }
4473 
4474 /*@
4475   DMPlexCreateFromCellListParallelPetsc - Create distributed DMPLEX from a list of vertices for each cell (common mesh generator output)
4476 
4477   Input Parameters:
4478 + comm - The communicator
4479 . dim - The topological dimension of the mesh
4480 . numCells - The number of cells owned by this process
4481 . numVertices - The number of vertices owned by this process, or PETSC_DECIDE
4482 . NVertices - The global number of vertices, or PETSC_DECIDE
4483 . numCorners - The number of vertices for each cell
4484 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4485 . cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4486 . spaceDim - The spatial dimension used for coordinates
4487 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4488 
4489   Output Parameters:
4490 + dm - The DM
4491 . vertexSF - (Optional) SF describing complete vertex ownership
4492 - verticesAdjSaved - (Optional) vertex adjacency array
4493 
4494   Notes:
4495   This function is just a convenient sequence of DMCreate(), DMSetType(), DMSetDimension(),
4496   DMPlexBuildFromCellListParallel(), DMPlexInterpolate(), DMPlexBuildCoordinatesFromCellListParallel()
4497 
4498   See DMPlexBuildFromCellListParallel() for an example and details about the topology-related parameters.
4499   See DMPlexBuildCoordinatesFromCellListParallel() for details about the geometry-related parameters.
4500 
4501   Level: intermediate
4502 
4503 .seealso: `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
4504 @*/
4505 PetscErrorCode DMPlexCreateFromCellListParallelPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, PetscInt **verticesAdj, DM *dm) {
4506   PetscSF sfVert;
4507 
4508   PetscFunctionBegin;
4509   PetscCall(DMCreate(comm, dm));
4510   PetscCall(DMSetType(*dm, DMPLEX));
4511   PetscValidLogicalCollectiveInt(*dm, dim, 2);
4512   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
4513   PetscCall(DMSetDimension(*dm, dim));
4514   PetscCall(DMPlexBuildFromCellListParallel(*dm, numCells, numVertices, NVertices, numCorners, cells, &sfVert, verticesAdj));
4515   if (interpolate) {
4516     DM idm;
4517 
4518     PetscCall(DMPlexInterpolate(*dm, &idm));
4519     PetscCall(DMDestroy(dm));
4520     *dm = idm;
4521   }
4522   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
4523   if (vertexSF) *vertexSF = sfVert;
4524   else PetscCall(PetscSFDestroy(&sfVert));
4525   PetscFunctionReturn(0);
4526 }
4527 
4528 /*@C
4529   DMPlexBuildFromCellList - Build DMPLEX topology from a list of vertices for each cell (common mesh generator output)
4530 
4531   Input Parameters:
4532 + dm - The DM
4533 . numCells - The number of cells owned by this process
4534 . numVertices - The number of vertices owned by this process, or PETSC_DETERMINE
4535 . numCorners - The number of vertices for each cell
4536 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4537 
4538   Level: advanced
4539 
4540   Notes:
4541   Two triangles sharing a face
4542 $
4543 $        2
4544 $      / | \
4545 $     /  |  \
4546 $    /   |   \
4547 $   0  0 | 1  3
4548 $    \   |   /
4549 $     \  |  /
4550 $      \ | /
4551 $        1
4552 would have input
4553 $  numCells = 2, numVertices = 4
4554 $  cells = [0 1 2  1 3 2]
4555 $
4556 which would result in the DMPlex
4557 $
4558 $        4
4559 $      / | \
4560 $     /  |  \
4561 $    /   |   \
4562 $   2  0 | 1  5
4563 $    \   |   /
4564 $     \  |  /
4565 $      \ | /
4566 $        3
4567 
4568   If numVertices is PETSC_DETERMINE, it is computed by PETSc as the maximum vertex index in cells + 1.
4569 
4570   Not currently supported in Fortran.
4571 
4572 .seealso: `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListPetsc()`
4573 @*/
4574 PetscErrorCode DMPlexBuildFromCellList(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const PetscInt cells[]) {
4575   PetscInt *cones, c, p, dim;
4576 
4577   PetscFunctionBegin;
4578   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4579   PetscCall(DMGetDimension(dm, &dim));
4580   /* Get/check global number of vertices */
4581   {
4582     PetscInt       NVerticesInCells, i;
4583     const PetscInt len = numCells * numCorners;
4584 
4585     /* NVerticesInCells = max(cells) + 1 */
4586     NVerticesInCells = PETSC_MIN_INT;
4587     for (i = 0; i < len; i++)
4588       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4589     ++NVerticesInCells;
4590 
4591     if (numVertices == PETSC_DECIDE) numVertices = NVerticesInCells;
4592     else
4593       PetscCheck(numVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, numVertices, NVerticesInCells);
4594   }
4595   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
4596   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
4597   PetscCall(DMSetUp(dm));
4598   PetscCall(DMPlexGetCones(dm, &cones));
4599   for (c = 0; c < numCells; ++c) {
4600     for (p = 0; p < numCorners; ++p) cones[c * numCorners + p] = cells[c * numCorners + p] + numCells;
4601   }
4602   PetscCall(DMPlexSymmetrize(dm));
4603   PetscCall(DMPlexStratify(dm));
4604   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4605   PetscFunctionReturn(0);
4606 }
4607 
4608 /*@C
4609   DMPlexBuildCoordinatesFromCellList - Build DM coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4610 
4611   Input Parameters:
4612 + dm - The DM
4613 . spaceDim - The spatial dimension used for coordinates
4614 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4615 
4616   Level: advanced
4617 
4618   Notes:
4619   Not currently supported in Fortran.
4620 
4621 .seealso: `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellList()`
4622 @*/
4623 PetscErrorCode DMPlexBuildCoordinatesFromCellList(DM dm, PetscInt spaceDim, const PetscReal vertexCoords[]) {
4624   PetscSection coordSection;
4625   Vec          coordinates;
4626   DM           cdm;
4627   PetscScalar *coords;
4628   PetscInt     v, vStart, vEnd, d;
4629 
4630   PetscFunctionBegin;
4631   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4632   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
4633   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4634   PetscCall(DMSetCoordinateDim(dm, spaceDim));
4635   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4636   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4637   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
4638   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
4639   for (v = vStart; v < vEnd; ++v) {
4640     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
4641     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
4642   }
4643   PetscCall(PetscSectionSetUp(coordSection));
4644 
4645   PetscCall(DMGetCoordinateDM(dm, &cdm));
4646   PetscCall(DMCreateLocalVector(cdm, &coordinates));
4647   PetscCall(VecSetBlockSize(coordinates, spaceDim));
4648   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4649   PetscCall(VecGetArrayWrite(coordinates, &coords));
4650   for (v = 0; v < vEnd - vStart; ++v) {
4651     for (d = 0; d < spaceDim; ++d) coords[v * spaceDim + d] = vertexCoords[v * spaceDim + d];
4652   }
4653   PetscCall(VecRestoreArrayWrite(coordinates, &coords));
4654   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4655   PetscCall(VecDestroy(&coordinates));
4656   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4657   PetscFunctionReturn(0);
4658 }
4659 
4660 /*@
4661   DMPlexCreateFromCellListPetsc - Create DMPLEX from a list of vertices for each cell (common mesh generator output), but only process 0 takes in the input
4662 
4663   Collective on comm
4664 
4665   Input Parameters:
4666 + comm - The communicator
4667 . dim - The topological dimension of the mesh
4668 . numCells - The number of cells, only on process 0
4669 . numVertices - The number of vertices owned by this process, or PETSC_DECIDE, only on process 0
4670 . numCorners - The number of vertices for each cell, only on process 0
4671 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4672 . cells - An array of numCells*numCorners numbers, the vertices for each cell, only on process 0
4673 . spaceDim - The spatial dimension used for coordinates
4674 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex, only on process 0
4675 
4676   Output Parameter:
4677 . dm - The DM, which only has points on process 0
4678 
4679   Notes:
4680   This function is just a convenient sequence of DMCreate(), DMSetType(), DMSetDimension(), DMPlexBuildFromCellList(),
4681   DMPlexInterpolate(), DMPlexBuildCoordinatesFromCellList()
4682 
4683   See DMPlexBuildFromCellList() for an example and details about the topology-related parameters.
4684   See DMPlexBuildCoordinatesFromCellList() for details about the geometry-related parameters.
4685   See DMPlexCreateFromCellListParallelPetsc() for parallel input
4686 
4687   Level: intermediate
4688 
4689 .seealso: `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellList()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
4690 @*/
4691 PetscErrorCode DMPlexCreateFromCellListPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], DM *dm) {
4692   PetscMPIInt rank;
4693 
4694   PetscFunctionBegin;
4695   PetscCheck(dim, comm, PETSC_ERR_ARG_OUTOFRANGE, "This is not appropriate for 0-dimensional meshes. Consider either creating the DM using DMPlexCreateFromDAG(), by hand, or using DMSwarm.");
4696   PetscCallMPI(MPI_Comm_rank(comm, &rank));
4697   PetscCall(DMCreate(comm, dm));
4698   PetscCall(DMSetType(*dm, DMPLEX));
4699   PetscCall(DMSetDimension(*dm, dim));
4700   if (rank == 0) PetscCall(DMPlexBuildFromCellList(*dm, numCells, numVertices, numCorners, cells));
4701   else PetscCall(DMPlexBuildFromCellList(*dm, 0, 0, 0, NULL));
4702   if (interpolate) {
4703     DM idm;
4704 
4705     PetscCall(DMPlexInterpolate(*dm, &idm));
4706     PetscCall(DMDestroy(dm));
4707     *dm = idm;
4708   }
4709   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, vertexCoords));
4710   else PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, NULL));
4711   PetscFunctionReturn(0);
4712 }
4713 
4714 /*@
4715   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a DM
4716 
4717   Input Parameters:
4718 + dm - The empty DM object, usually from DMCreate() and DMSetDimension()
4719 . depth - The depth of the DAG
4720 . numPoints - Array of size depth + 1 containing the number of points at each depth
4721 . coneSize - The cone size of each point
4722 . cones - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
4723 . coneOrientations - The orientation of each cone point
4724 - vertexCoords - An array of numPoints[0]*spacedim numbers representing the coordinates of each vertex, with spacedim the value set via DMSetCoordinateDim()
4725 
4726   Output Parameter:
4727 . dm - The DM
4728 
4729   Note: Two triangles sharing a face would have input
4730 $  depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
4731 $  cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
4732 $ vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
4733 $
4734 which would result in the DMPlex
4735 $
4736 $        4
4737 $      / | \
4738 $     /  |  \
4739 $    /   |   \
4740 $   2  0 | 1  5
4741 $    \   |   /
4742 $     \  |  /
4743 $      \ | /
4744 $        3
4745 $
4746 $ Notice that all points are numbered consecutively, unlike DMPlexCreateFromCellListPetsc()
4747 
4748   Level: advanced
4749 
4750 .seealso: `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
4751 @*/
4752 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[]) {
4753   Vec          coordinates;
4754   PetscSection coordSection;
4755   PetscScalar *coords;
4756   PetscInt     coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
4757 
4758   PetscFunctionBegin;
4759   PetscCall(DMGetDimension(dm, &dim));
4760   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
4761   PetscCheck(dimEmbed >= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Embedding dimension %" PetscInt_FMT " cannot be less than intrinsic dimension %" PetscInt_FMT, dimEmbed, dim);
4762   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
4763   PetscCall(DMPlexSetChart(dm, pStart, pEnd));
4764   for (p = pStart; p < pEnd; ++p) {
4765     PetscCall(DMPlexSetConeSize(dm, p, coneSize[p - pStart]));
4766     if (firstVertex < 0 && !coneSize[p - pStart]) firstVertex = p - pStart;
4767   }
4768   PetscCheck(firstVertex >= 0 || !numPoints[0], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Expected %" PetscInt_FMT " vertices but could not find any", numPoints[0]);
4769   PetscCall(DMSetUp(dm)); /* Allocate space for cones */
4770   for (p = pStart, off = 0; p < pEnd; off += coneSize[p - pStart], ++p) {
4771     PetscCall(DMPlexSetCone(dm, p, &cones[off]));
4772     PetscCall(DMPlexSetConeOrientation(dm, p, &coneOrientations[off]));
4773   }
4774   PetscCall(DMPlexSymmetrize(dm));
4775   PetscCall(DMPlexStratify(dm));
4776   /* Build coordinates */
4777   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4778   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4779   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dimEmbed));
4780   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numPoints[0]));
4781   for (v = firstVertex; v < firstVertex + numPoints[0]; ++v) {
4782     PetscCall(PetscSectionSetDof(coordSection, v, dimEmbed));
4783     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed));
4784   }
4785   PetscCall(PetscSectionSetUp(coordSection));
4786   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4787   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
4788   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4789   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4790   PetscCall(VecSetBlockSize(coordinates, dimEmbed));
4791   PetscCall(VecSetType(coordinates, VECSTANDARD));
4792   if (vertexCoords) {
4793     PetscCall(VecGetArray(coordinates, &coords));
4794     for (v = 0; v < numPoints[0]; ++v) {
4795       PetscInt off;
4796 
4797       PetscCall(PetscSectionGetOffset(coordSection, v + firstVertex, &off));
4798       for (d = 0; d < dimEmbed; ++d) coords[off + d] = vertexCoords[v * dimEmbed + d];
4799     }
4800   }
4801   PetscCall(VecRestoreArray(coordinates, &coords));
4802   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4803   PetscCall(VecDestroy(&coordinates));
4804   PetscFunctionReturn(0);
4805 }
4806 
4807 /*@C
4808   DMPlexCreateCellVertexFromFile - Create a DMPlex mesh from a simple cell-vertex file.
4809 
4810 + comm        - The MPI communicator
4811 . filename    - Name of the .dat file
4812 - interpolate - Create faces and edges in the mesh
4813 
4814   Output Parameter:
4815 . dm  - The DM object representing the mesh
4816 
4817   Note: The format is the simplest possible:
4818 $ Ne
4819 $ v0 v1 ... vk
4820 $ Nv
4821 $ x y z marker
4822 
4823   Level: beginner
4824 
4825 .seealso: `DMPlexCreateFromFile()`, `DMPlexCreateMedFromFile()`, `DMPlexCreateGmsh()`, `DMPlexCreate()`
4826 @*/
4827 PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm) {
4828   DMLabel      marker;
4829   PetscViewer  viewer;
4830   Vec          coordinates;
4831   PetscSection coordSection;
4832   PetscScalar *coords;
4833   char         line[PETSC_MAX_PATH_LEN];
4834   PetscInt     dim = 3, cdim = 3, coordSize, v, c, d;
4835   PetscMPIInt  rank;
4836   int          snum, Nv, Nc, Ncn, Nl;
4837 
4838   PetscFunctionBegin;
4839   PetscCallMPI(MPI_Comm_rank(comm, &rank));
4840   PetscCall(PetscViewerCreate(comm, &viewer));
4841   PetscCall(PetscViewerSetType(viewer, PETSCVIEWERASCII));
4842   PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
4843   PetscCall(PetscViewerFileSetName(viewer, filename));
4844   if (rank == 0) {
4845     PetscCall(PetscViewerRead(viewer, line, 4, NULL, PETSC_STRING));
4846     snum = sscanf(line, "%d %d %d %d", &Nc, &Nv, &Ncn, &Nl);
4847     PetscCheck(snum == 4, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4848   } else {
4849     Nc = Nv = Ncn = Nl = 0;
4850   }
4851   PetscCall(DMCreate(comm, dm));
4852   PetscCall(DMSetType(*dm, DMPLEX));
4853   PetscCall(DMPlexSetChart(*dm, 0, Nc + Nv));
4854   PetscCall(DMSetDimension(*dm, dim));
4855   PetscCall(DMSetCoordinateDim(*dm, cdim));
4856   /* Read topology */
4857   if (rank == 0) {
4858     char     format[PETSC_MAX_PATH_LEN];
4859     PetscInt cone[8];
4860     int      vbuf[8], v;
4861 
4862     for (c = 0; c < Ncn; ++c) {
4863       format[c * 3 + 0] = '%';
4864       format[c * 3 + 1] = 'd';
4865       format[c * 3 + 2] = ' ';
4866     }
4867     format[Ncn * 3 - 1] = '\0';
4868     for (c = 0; c < Nc; ++c) PetscCall(DMPlexSetConeSize(*dm, c, Ncn));
4869     PetscCall(DMSetUp(*dm));
4870     for (c = 0; c < Nc; ++c) {
4871       PetscCall(PetscViewerRead(viewer, line, Ncn, NULL, PETSC_STRING));
4872       switch (Ncn) {
4873       case 2: snum = sscanf(line, format, &vbuf[0], &vbuf[1]); break;
4874       case 3: snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2]); break;
4875       case 4: snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3]); break;
4876       case 6: snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5]); break;
4877       case 8: snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]); break;
4878       default: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No cell shape with %d vertices", Ncn);
4879       }
4880       PetscCheck(snum == Ncn, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4881       for (v = 0; v < Ncn; ++v) cone[v] = vbuf[v] + Nc;
4882       /* Hexahedra are inverted */
4883       if (Ncn == 8) {
4884         PetscInt tmp = cone[1];
4885         cone[1]      = cone[3];
4886         cone[3]      = tmp;
4887       }
4888       PetscCall(DMPlexSetCone(*dm, c, cone));
4889     }
4890   }
4891   PetscCall(DMPlexSymmetrize(*dm));
4892   PetscCall(DMPlexStratify(*dm));
4893   /* Read coordinates */
4894   PetscCall(DMGetCoordinateSection(*dm, &coordSection));
4895   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4896   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
4897   PetscCall(PetscSectionSetChart(coordSection, Nc, Nc + Nv));
4898   for (v = Nc; v < Nc + Nv; ++v) {
4899     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
4900     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
4901   }
4902   PetscCall(PetscSectionSetUp(coordSection));
4903   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4904   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
4905   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4906   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4907   PetscCall(VecSetBlockSize(coordinates, cdim));
4908   PetscCall(VecSetType(coordinates, VECSTANDARD));
4909   PetscCall(VecGetArray(coordinates, &coords));
4910   if (rank == 0) {
4911     char   format[PETSC_MAX_PATH_LEN];
4912     double x[3];
4913     int    l, val[3];
4914 
4915     if (Nl) {
4916       for (l = 0; l < Nl; ++l) {
4917         format[l * 3 + 0] = '%';
4918         format[l * 3 + 1] = 'd';
4919         format[l * 3 + 2] = ' ';
4920       }
4921       format[Nl * 3 - 1] = '\0';
4922       PetscCall(DMCreateLabel(*dm, "marker"));
4923       PetscCall(DMGetLabel(*dm, "marker", &marker));
4924     }
4925     for (v = 0; v < Nv; ++v) {
4926       PetscCall(PetscViewerRead(viewer, line, 3 + Nl, NULL, PETSC_STRING));
4927       snum = sscanf(line, "%lg %lg %lg", &x[0], &x[1], &x[2]);
4928       PetscCheck(snum == 3, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4929       switch (Nl) {
4930       case 0: snum = 0; break;
4931       case 1: snum = sscanf(line, format, &val[0]); break;
4932       case 2: snum = sscanf(line, format, &val[0], &val[1]); break;
4933       case 3: snum = sscanf(line, format, &val[0], &val[1], &val[2]); break;
4934       default: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Request support for %d labels", Nl);
4935       }
4936       PetscCheck(snum == Nl, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4937       for (d = 0; d < cdim; ++d) coords[v * cdim + d] = x[d];
4938       for (l = 0; l < Nl; ++l) PetscCall(DMLabelSetValue(marker, v + Nc, val[l]));
4939     }
4940   }
4941   PetscCall(VecRestoreArray(coordinates, &coords));
4942   PetscCall(DMSetCoordinatesLocal(*dm, coordinates));
4943   PetscCall(VecDestroy(&coordinates));
4944   PetscCall(PetscViewerDestroy(&viewer));
4945   if (interpolate) {
4946     DM      idm;
4947     DMLabel bdlabel;
4948 
4949     PetscCall(DMPlexInterpolate(*dm, &idm));
4950     PetscCall(DMDestroy(dm));
4951     *dm = idm;
4952 
4953     if (!Nl) {
4954       PetscCall(DMCreateLabel(*dm, "marker"));
4955       PetscCall(DMGetLabel(*dm, "marker", &bdlabel));
4956       PetscCall(DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel));
4957       PetscCall(DMPlexLabelComplete(*dm, bdlabel));
4958     }
4959   }
4960   PetscFunctionReturn(0);
4961 }
4962 
4963 /*@C
4964   DMPlexCreateFromFile - This takes a filename and produces a DM
4965 
4966   Input Parameters:
4967 + comm - The communicator
4968 . filename - A file name
4969 . plexname - The object name of the resulting DM, also used for intra-datafile lookup by some formats
4970 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
4971 
4972   Output Parameter:
4973 . dm - The DM
4974 
4975   Options Database Keys:
4976 . -dm_plex_create_from_hdf5_xdmf - use the PETSC_VIEWER_HDF5_XDMF format for reading HDF5
4977 
4978   Use -dm_plex_create_ prefix to pass options to the internal PetscViewer, e.g.
4979 $ -dm_plex_create_viewer_hdf5_collective
4980 
4981   Notes:
4982   Using PETSCVIEWERHDF5 type with PETSC_VIEWER_HDF5_PETSC format, one can save multiple DMPlex
4983   meshes in a single HDF5 file. This in turn requires one to name the DMPlex object with PetscObjectSetName()
4984   before saving it with DMView() and before loading it with DMLoad() for identification of the mesh object.
4985   The input parameter name is thus used to name the DMPlex object when DMPlexCreateFromFile() internally
4986   calls DMLoad(). Currently, name is ignored for other viewer types and/or formats.
4987 
4988   Level: beginner
4989 
4990 .seealso: `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`, `PetscObjectSetName()`, `DMView()`, `DMLoad()`
4991 @*/
4992 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], const char plexname[], PetscBool interpolate, DM *dm) {
4993   const char  extGmsh[]      = ".msh";
4994   const char  extGmsh2[]     = ".msh2";
4995   const char  extGmsh4[]     = ".msh4";
4996   const char  extCGNS[]      = ".cgns";
4997   const char  extExodus[]    = ".exo";
4998   const char  extExodus_e[]  = ".e";
4999   const char  extGenesis[]   = ".gen";
5000   const char  extFluent[]    = ".cas";
5001   const char  extHDF5[]      = ".h5";
5002   const char  extMed[]       = ".med";
5003   const char  extPLY[]       = ".ply";
5004   const char  extEGADSLite[] = ".egadslite";
5005   const char  extEGADS[]     = ".egads";
5006   const char  extIGES[]      = ".igs";
5007   const char  extSTEP[]      = ".stp";
5008   const char  extCV[]        = ".dat";
5009   size_t      len;
5010   PetscBool   isGmsh, isGmsh2, isGmsh4, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isMed, isPLY, isEGADSLite, isEGADS, isIGES, isSTEP, isCV;
5011   PetscMPIInt rank;
5012 
5013   PetscFunctionBegin;
5014   PetscValidCharPointer(filename, 2);
5015   if (plexname) PetscValidCharPointer(plexname, 3);
5016   PetscValidPointer(dm, 5);
5017   PetscCall(DMInitializePackage());
5018   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5019   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5020   PetscCall(PetscStrlen(filename, &len));
5021   PetscCheck(len, comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
5022 
5023 #define CheckExtension(extension__, is_extension__) \
5024   do { \
5025     PetscAssert(sizeof(extension__), comm, PETSC_ERR_PLIB, "Zero-size extension: %s", extension__); \
5026     /* don't count the null-terminator at the end */ \
5027     const size_t ext_len = sizeof(extension__) - 1; \
5028     if (len < ext_len) { \
5029       is_extension__ = PETSC_FALSE; \
5030     } else { \
5031       PetscCall(PetscStrncmp(filename + len - ext_len, extension__, ext_len, &is_extension__)); \
5032     } \
5033   } while (0)
5034 
5035   CheckExtension(extGmsh, isGmsh);
5036   CheckExtension(extGmsh2, isGmsh2);
5037   CheckExtension(extGmsh4, isGmsh4);
5038   CheckExtension(extCGNS, isCGNS);
5039   CheckExtension(extExodus, isExodus);
5040   if (!isExodus) CheckExtension(extExodus_e, isExodus);
5041   CheckExtension(extGenesis, isGenesis);
5042   CheckExtension(extFluent, isFluent);
5043   CheckExtension(extHDF5, isHDF5);
5044   CheckExtension(extMed, isMed);
5045   CheckExtension(extPLY, isPLY);
5046   CheckExtension(extEGADSLite, isEGADSLite);
5047   CheckExtension(extEGADS, isEGADS);
5048   CheckExtension(extIGES, isIGES);
5049   CheckExtension(extSTEP, isSTEP);
5050   CheckExtension(extCV, isCV);
5051 
5052 #undef CheckExtension
5053 
5054   if (isGmsh || isGmsh2 || isGmsh4) {
5055     PetscCall(DMPlexCreateGmshFromFile(comm, filename, interpolate, dm));
5056   } else if (isCGNS) {
5057     PetscCall(DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm));
5058   } else if (isExodus || isGenesis) {
5059     PetscCall(DMPlexCreateExodusFromFile(comm, filename, interpolate, dm));
5060   } else if (isFluent) {
5061     PetscCall(DMPlexCreateFluentFromFile(comm, filename, interpolate, dm));
5062   } else if (isHDF5) {
5063     PetscBool   load_hdf5_xdmf = PETSC_FALSE;
5064     PetscViewer viewer;
5065 
5066     /* PETSC_VIEWER_HDF5_XDMF is used if the filename ends with .xdmf.h5, or if -dm_plex_create_from_hdf5_xdmf option is present */
5067     PetscCall(PetscStrncmp(&filename[PetscMax(0, len - 8)], ".xdmf", 5, &load_hdf5_xdmf));
5068     PetscCall(PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &load_hdf5_xdmf, NULL));
5069     PetscCall(PetscViewerCreate(comm, &viewer));
5070     PetscCall(PetscViewerSetType(viewer, PETSCVIEWERHDF5));
5071     PetscCall(PetscViewerSetOptionsPrefix(viewer, "dm_plex_create_"));
5072     PetscCall(PetscViewerSetFromOptions(viewer));
5073     PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
5074     PetscCall(PetscViewerFileSetName(viewer, filename));
5075 
5076     PetscCall(DMCreate(comm, dm));
5077     PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5078     PetscCall(DMSetType(*dm, DMPLEX));
5079     if (load_hdf5_xdmf) PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF));
5080     PetscCall(DMLoad(*dm, viewer));
5081     if (load_hdf5_xdmf) PetscCall(PetscViewerPopFormat(viewer));
5082     PetscCall(PetscViewerDestroy(&viewer));
5083 
5084     if (interpolate) {
5085       DM idm;
5086 
5087       PetscCall(DMPlexInterpolate(*dm, &idm));
5088       PetscCall(DMDestroy(dm));
5089       *dm = idm;
5090     }
5091   } else if (isMed) {
5092     PetscCall(DMPlexCreateMedFromFile(comm, filename, interpolate, dm));
5093   } else if (isPLY) {
5094     PetscCall(DMPlexCreatePLYFromFile(comm, filename, interpolate, dm));
5095   } else if (isEGADSLite || isEGADS || isIGES || isSTEP) {
5096     if (isEGADSLite) PetscCall(DMPlexCreateEGADSLiteFromFile(comm, filename, dm));
5097     else PetscCall(DMPlexCreateEGADSFromFile(comm, filename, dm));
5098     if (!interpolate) {
5099       DM udm;
5100 
5101       PetscCall(DMPlexUninterpolate(*dm, &udm));
5102       PetscCall(DMDestroy(dm));
5103       *dm = udm;
5104     }
5105   } else if (isCV) {
5106     PetscCall(DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm));
5107   } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
5108   PetscCall(PetscStrlen(plexname, &len));
5109   if (len) PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5110   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5111   PetscFunctionReturn(0);
5112 }
5113