xref: /petsc/src/dm/impls/plex/plexcreate.c (revision 217f7fc6b7ccbb041bf230fae922d6b0eefae257)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h> /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>   /*I   "petscdmplex.h"   I*/
4 #include <petscsf.h>
5 #include <petscdmplextransform.h>
6 #include <petscdmlabelephemeral.h>
7 #include <petsc/private/kernels/blockmatmult.h>
8 #include <petsc/private/kernels/blockinvert.h>
9 
10 PetscLogEvent DMPLEX_CreateFromFile, DMPLEX_BuildFromCellList, DMPLEX_BuildCoordinatesFromCellList;
11 
12 /* External function declarations here */
13 static PetscErrorCode DMInitialize_Plex(DM dm);
14 
15 /* This copies internal things in the Plex structure that we generally want when making a new, related Plex */
16 PetscErrorCode DMPlexCopy_Internal(DM dmin, PetscBool copyPeriodicity, PetscBool copyOverlap, DM dmout)
17 {
18   const PetscReal         *maxCell, *Lstart, *L;
19   PetscBool                dist;
20   DMPlexReorderDefaultFlag reorder;
21 
22   PetscFunctionBegin;
23   if (copyPeriodicity) {
24     PetscCall(DMGetPeriodicity(dmin, &maxCell, &Lstart, &L));
25     PetscCall(DMSetPeriodicity(dmout, maxCell, Lstart, L));
26   }
27   PetscCall(DMPlexDistributeGetDefault(dmin, &dist));
28   PetscCall(DMPlexDistributeSetDefault(dmout, dist));
29   PetscCall(DMPlexReorderGetDefault(dmin, &reorder));
30   PetscCall(DMPlexReorderSetDefault(dmout, reorder));
31   ((DM_Plex *)dmout->data)->useHashLocation = ((DM_Plex *)dmin->data)->useHashLocation;
32   if (copyOverlap) PetscCall(DMPlexSetOverlap_Plex(dmout, dmin, 0));
33   PetscFunctionReturn(0);
34 }
35 
36 /* Replace dm with the contents of ndm, and then destroy ndm
37    - Share the DM_Plex structure
38    - Share the coordinates
39    - Share the SF
40 */
41 PetscErrorCode DMPlexReplace_Internal(DM dm, DM *ndm)
42 {
43   PetscSF          sf;
44   DM               dmNew = *ndm, coordDM, coarseDM;
45   Vec              coords;
46   const PetscReal *maxCell, *Lstart, *L;
47   PetscInt         dim, cdim;
48 
49   PetscFunctionBegin;
50   if (dm == dmNew) {
51     PetscCall(DMDestroy(ndm));
52     PetscFunctionReturn(0);
53   }
54   dm->setupcalled = dmNew->setupcalled;
55   PetscCall(DMGetDimension(dmNew, &dim));
56   PetscCall(DMSetDimension(dm, dim));
57   PetscCall(DMGetCoordinateDim(dmNew, &cdim));
58   PetscCall(DMSetCoordinateDim(dm, cdim));
59   PetscCall(DMGetPointSF(dmNew, &sf));
60   PetscCall(DMSetPointSF(dm, sf));
61   PetscCall(DMGetCoordinateDM(dmNew, &coordDM));
62   PetscCall(DMGetCoordinatesLocal(dmNew, &coords));
63   PetscCall(DMSetCoordinateDM(dm, coordDM));
64   PetscCall(DMSetCoordinatesLocal(dm, coords));
65   PetscCall(DMGetCellCoordinateDM(dmNew, &coordDM));
66   PetscCall(DMGetCellCoordinatesLocal(dmNew, &coords));
67   PetscCall(DMSetCellCoordinateDM(dm, coordDM));
68   PetscCall(DMSetCellCoordinatesLocal(dm, coords));
69   /* Do not want to create the coordinate field if it does not already exist, so do not call DMGetCoordinateField() */
70   PetscCall(DMFieldDestroy(&dm->coordinates[0].field));
71   dm->coordinates[0].field            = dmNew->coordinates[0].field;
72   ((DM_Plex *)dmNew->data)->coordFunc = ((DM_Plex *)dm->data)->coordFunc;
73   PetscCall(DMGetPeriodicity(dmNew, &maxCell, &Lstart, &L));
74   PetscCall(DMSetPeriodicity(dm, maxCell, Lstart, L));
75   PetscCall(DMDestroy_Plex(dm));
76   PetscCall(DMInitialize_Plex(dm));
77   dm->data = dmNew->data;
78   ((DM_Plex *)dmNew->data)->refct++;
79   PetscCall(DMDestroyLabelLinkList_Internal(dm));
80   PetscCall(DMCopyLabels(dmNew, dm, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL));
81   PetscCall(DMGetCoarseDM(dmNew, &coarseDM));
82   PetscCall(DMSetCoarseDM(dm, coarseDM));
83   PetscCall(DMDestroy(ndm));
84   PetscFunctionReturn(0);
85 }
86 
87 /* Swap dm with the contents of dmNew
88    - Swap the DM_Plex structure
89    - Swap the coordinates
90    - Swap the point PetscSF
91 */
92 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
93 {
94   DM          coordDMA, coordDMB;
95   Vec         coordsA, coordsB;
96   PetscSF     sfA, sfB;
97   DMField     fieldTmp;
98   void       *tmp;
99   DMLabelLink listTmp;
100   DMLabel     depthTmp;
101   PetscInt    tmpI;
102 
103   PetscFunctionBegin;
104   if (dmA == dmB) PetscFunctionReturn(0);
105   PetscCall(DMGetPointSF(dmA, &sfA));
106   PetscCall(DMGetPointSF(dmB, &sfB));
107   PetscCall(PetscObjectReference((PetscObject)sfA));
108   PetscCall(DMSetPointSF(dmA, sfB));
109   PetscCall(DMSetPointSF(dmB, sfA));
110   PetscCall(PetscObjectDereference((PetscObject)sfA));
111 
112   PetscCall(DMGetCoordinateDM(dmA, &coordDMA));
113   PetscCall(DMGetCoordinateDM(dmB, &coordDMB));
114   PetscCall(PetscObjectReference((PetscObject)coordDMA));
115   PetscCall(DMSetCoordinateDM(dmA, coordDMB));
116   PetscCall(DMSetCoordinateDM(dmB, coordDMA));
117   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
118 
119   PetscCall(DMGetCoordinatesLocal(dmA, &coordsA));
120   PetscCall(DMGetCoordinatesLocal(dmB, &coordsB));
121   PetscCall(PetscObjectReference((PetscObject)coordsA));
122   PetscCall(DMSetCoordinatesLocal(dmA, coordsB));
123   PetscCall(DMSetCoordinatesLocal(dmB, coordsA));
124   PetscCall(PetscObjectDereference((PetscObject)coordsA));
125 
126   PetscCall(DMGetCellCoordinateDM(dmA, &coordDMA));
127   PetscCall(DMGetCellCoordinateDM(dmB, &coordDMB));
128   PetscCall(PetscObjectReference((PetscObject)coordDMA));
129   PetscCall(DMSetCellCoordinateDM(dmA, coordDMB));
130   PetscCall(DMSetCellCoordinateDM(dmB, coordDMA));
131   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
132 
133   PetscCall(DMGetCellCoordinatesLocal(dmA, &coordsA));
134   PetscCall(DMGetCellCoordinatesLocal(dmB, &coordsB));
135   PetscCall(PetscObjectReference((PetscObject)coordsA));
136   PetscCall(DMSetCellCoordinatesLocal(dmA, coordsB));
137   PetscCall(DMSetCellCoordinatesLocal(dmB, coordsA));
138   PetscCall(PetscObjectDereference((PetscObject)coordsA));
139 
140   fieldTmp                  = dmA->coordinates[0].field;
141   dmA->coordinates[0].field = dmB->coordinates[0].field;
142   dmB->coordinates[0].field = fieldTmp;
143   fieldTmp                  = dmA->coordinates[1].field;
144   dmA->coordinates[1].field = dmB->coordinates[1].field;
145   dmB->coordinates[1].field = fieldTmp;
146   tmp                       = dmA->data;
147   dmA->data                 = dmB->data;
148   dmB->data                 = tmp;
149   listTmp                   = dmA->labels;
150   dmA->labels               = dmB->labels;
151   dmB->labels               = listTmp;
152   depthTmp                  = dmA->depthLabel;
153   dmA->depthLabel           = dmB->depthLabel;
154   dmB->depthLabel           = depthTmp;
155   depthTmp                  = dmA->celltypeLabel;
156   dmA->celltypeLabel        = dmB->celltypeLabel;
157   dmB->celltypeLabel        = depthTmp;
158   tmpI                      = dmA->levelup;
159   dmA->levelup              = dmB->levelup;
160   dmB->levelup              = tmpI;
161   PetscFunctionReturn(0);
162 }
163 
164 static PetscErrorCode DMPlexInterpolateInPlace_Internal(DM dm)
165 {
166   DM idm;
167 
168   PetscFunctionBegin;
169   PetscCall(DMPlexInterpolate(dm, &idm));
170   PetscCall(DMPlexCopyCoordinates(dm, idm));
171   PetscCall(DMPlexReplace_Internal(dm, &idm));
172   PetscFunctionReturn(0);
173 }
174 
175 /*@C
176   DMPlexCreateCoordinateSpace - Creates a finite element space for the coordinates
177 
178   Collective
179 
180   Input Parameters:
181 + DM        - The DM
182 . degree    - The degree of the finite element or PETSC_DECIDE
183 - coordFunc - An optional function to map new points from refinement to the surface
184 
185   Level: advanced
186 
187 .seealso: `PetscFECreateLagrange()`, `DMGetCoordinateDM()`
188 @*/
189 PetscErrorCode DMPlexCreateCoordinateSpace(DM dm, PetscInt degree, PetscPointFunc coordFunc)
190 {
191   DM_Plex     *mesh = (DM_Plex *)dm->data;
192   DM           cdm;
193   PetscDS      cds;
194   PetscFE      fe;
195   PetscClassId id;
196 
197   PetscFunctionBegin;
198   PetscCall(DMGetCoordinateDM(dm, &cdm));
199   PetscCall(DMGetDS(cdm, &cds));
200   PetscCall(PetscDSGetDiscretization(cds, 0, (PetscObject *)&fe));
201   PetscCall(PetscObjectGetClassId((PetscObject)fe, &id));
202   if (id != PETSCFE_CLASSID) {
203     PetscBool simplex;
204     PetscInt  dim, dE, qorder;
205 
206     PetscCall(DMGetDimension(dm, &dim));
207     PetscCall(DMGetCoordinateDim(dm, &dE));
208     qorder = degree;
209     PetscObjectOptionsBegin((PetscObject)cdm);
210     PetscCall(PetscOptionsBoundedInt("-coord_dm_default_quadrature_order", "Quadrature order is one less than quadrature points per edge", "DMPlexCreateCoordinateSpace", qorder, &qorder, NULL, 0));
211     PetscOptionsEnd();
212     if (degree == PETSC_DECIDE) fe = NULL;
213     else {
214       PetscCall(DMPlexIsSimplex(dm, &simplex));
215       PetscCall(PetscFECreateLagrange(PETSC_COMM_SELF, dim, dE, simplex, degree, qorder, &fe));
216     }
217     PetscCall(DMProjectCoordinates(dm, fe));
218     PetscCall(PetscFEDestroy(&fe));
219   }
220   mesh->coordFunc = coordFunc;
221   PetscFunctionReturn(0);
222 }
223 
224 /*@
225   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
226 
227   Collective
228 
229   Input Parameters:
230 + comm - The communicator for the DM object
231 . dim - The spatial dimension
232 . simplex - Flag for simplicial cells, otherwise they are tensor product cells
233 . interpolate - Flag to create intermediate mesh pieces (edges, faces)
234 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
235 
236   Output Parameter:
237 . dm - The DM object
238 
239   Level: beginner
240 
241 .seealso: `DMSetType()`, `DMCreate()`
242 @*/
243 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscReal refinementLimit, DM *newdm)
244 {
245   DM          dm;
246   PetscMPIInt rank;
247 
248   PetscFunctionBegin;
249   PetscCall(DMCreate(comm, &dm));
250   PetscCall(DMSetType(dm, DMPLEX));
251   PetscCall(DMSetDimension(dm, dim));
252   PetscCallMPI(MPI_Comm_rank(comm, &rank));
253   switch (dim) {
254   case 2:
255     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "triangular"));
256     else PetscCall(PetscObjectSetName((PetscObject)dm, "quadrilateral"));
257     break;
258   case 3:
259     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "tetrahedral"));
260     else PetscCall(PetscObjectSetName((PetscObject)dm, "hexahedral"));
261     break;
262   default:
263     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
264   }
265   if (rank) {
266     PetscInt numPoints[2] = {0, 0};
267     PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL));
268   } else {
269     switch (dim) {
270     case 2:
271       if (simplex) {
272         PetscInt    numPoints[2]        = {4, 2};
273         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
274         PetscInt    cones[6]            = {2, 3, 4, 5, 4, 3};
275         PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
276         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
277 
278         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
279       } else {
280         PetscInt    numPoints[2]        = {6, 2};
281         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
282         PetscInt    cones[8]            = {2, 3, 4, 5, 3, 6, 7, 4};
283         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
284         PetscScalar vertexCoords[12]    = {-1.0, -0.5, 0.0, -0.5, 0.0, 0.5, -1.0, 0.5, 1.0, -0.5, 1.0, 0.5};
285 
286         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
287       }
288       break;
289     case 3:
290       if (simplex) {
291         PetscInt    numPoints[2]        = {5, 2};
292         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
293         PetscInt    cones[8]            = {4, 3, 5, 2, 5, 3, 4, 6};
294         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
295         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0};
296 
297         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
298       } else {
299         PetscInt    numPoints[2]         = {12, 2};
300         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
301         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9, 5, 4, 10, 11, 7, 12, 13, 8};
302         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
303         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5, -1.0, 0.5, -0.5, 0.0, 0.5, -0.5, 0.0, -0.5, -0.5, -1.0, -0.5, 0.5, 0.0, -0.5, 0.5, 0.0, 0.5, 0.5, -1.0, 0.5, 0.5, 1.0, 0.5, -0.5, 1.0, -0.5, -0.5, 1.0, -0.5, 0.5, 1.0, 0.5, 0.5};
304 
305         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
306       }
307       break;
308     default:
309       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
310     }
311   }
312   *newdm = dm;
313   if (refinementLimit > 0.0) {
314     DM          rdm;
315     const char *name;
316 
317     PetscCall(DMPlexSetRefinementUniform(*newdm, PETSC_FALSE));
318     PetscCall(DMPlexSetRefinementLimit(*newdm, refinementLimit));
319     PetscCall(DMRefine(*newdm, comm, &rdm));
320     PetscCall(PetscObjectGetName((PetscObject)*newdm, &name));
321     PetscCall(PetscObjectSetName((PetscObject)rdm, name));
322     PetscCall(DMDestroy(newdm));
323     *newdm = rdm;
324   }
325   if (interpolate) {
326     DM idm;
327 
328     PetscCall(DMPlexInterpolate(*newdm, &idm));
329     PetscCall(DMDestroy(newdm));
330     *newdm = idm;
331   }
332   PetscFunctionReturn(0);
333 }
334 
335 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
336 {
337   const PetscInt numVertices    = 2;
338   PetscInt       markerRight    = 1;
339   PetscInt       markerLeft     = 1;
340   PetscBool      markerSeparate = PETSC_FALSE;
341   Vec            coordinates;
342   PetscSection   coordSection;
343   PetscScalar   *coords;
344   PetscInt       coordSize;
345   PetscMPIInt    rank;
346   PetscInt       cdim = 1, v;
347 
348   PetscFunctionBegin;
349   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
350   if (markerSeparate) {
351     markerRight = 2;
352     markerLeft  = 1;
353   }
354   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
355   if (rank == 0) {
356     PetscCall(DMPlexSetChart(dm, 0, numVertices));
357     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
358     PetscCall(DMSetLabelValue(dm, "marker", 0, markerLeft));
359     PetscCall(DMSetLabelValue(dm, "marker", 1, markerRight));
360   }
361   PetscCall(DMPlexSymmetrize(dm));
362   PetscCall(DMPlexStratify(dm));
363   /* Build coordinates */
364   PetscCall(DMSetCoordinateDim(dm, cdim));
365   PetscCall(DMGetCoordinateSection(dm, &coordSection));
366   PetscCall(PetscSectionSetNumFields(coordSection, 1));
367   PetscCall(PetscSectionSetChart(coordSection, 0, numVertices));
368   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
369   for (v = 0; v < numVertices; ++v) {
370     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
371     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
372   }
373   PetscCall(PetscSectionSetUp(coordSection));
374   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
375   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
376   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
377   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
378   PetscCall(VecSetBlockSize(coordinates, cdim));
379   PetscCall(VecSetType(coordinates, VECSTANDARD));
380   PetscCall(VecGetArray(coordinates, &coords));
381   coords[0] = lower[0];
382   coords[1] = upper[0];
383   PetscCall(VecRestoreArray(coordinates, &coords));
384   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
385   PetscCall(VecDestroy(&coordinates));
386   PetscFunctionReturn(0);
387 }
388 
389 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
390 {
391   const PetscInt numVertices    = (edges[0] + 1) * (edges[1] + 1);
392   const PetscInt numEdges       = edges[0] * (edges[1] + 1) + (edges[0] + 1) * edges[1];
393   PetscInt       markerTop      = 1;
394   PetscInt       markerBottom   = 1;
395   PetscInt       markerRight    = 1;
396   PetscInt       markerLeft     = 1;
397   PetscBool      markerSeparate = PETSC_FALSE;
398   Vec            coordinates;
399   PetscSection   coordSection;
400   PetscScalar   *coords;
401   PetscInt       coordSize;
402   PetscMPIInt    rank;
403   PetscInt       v, vx, vy;
404 
405   PetscFunctionBegin;
406   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
407   if (markerSeparate) {
408     markerTop    = 3;
409     markerBottom = 1;
410     markerRight  = 2;
411     markerLeft   = 4;
412   }
413   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
414   if (rank == 0) {
415     PetscInt e, ex, ey;
416 
417     PetscCall(DMPlexSetChart(dm, 0, numEdges + numVertices));
418     for (e = 0; e < numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
419     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
420     for (vx = 0; vx <= edges[0]; vx++) {
421       for (ey = 0; ey < edges[1]; ey++) {
422         PetscInt edge   = vx * edges[1] + ey + edges[0] * (edges[1] + 1);
423         PetscInt vertex = ey * (edges[0] + 1) + vx + numEdges;
424         PetscInt cone[2];
425 
426         cone[0] = vertex;
427         cone[1] = vertex + edges[0] + 1;
428         PetscCall(DMPlexSetCone(dm, edge, cone));
429         if (vx == edges[0]) {
430           PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
431           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
432           if (ey == edges[1] - 1) {
433             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
434             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerRight));
435           }
436         } else if (vx == 0) {
437           PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
438           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
439           if (ey == edges[1] - 1) {
440             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
441             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft));
442           }
443         }
444       }
445     }
446     for (vy = 0; vy <= edges[1]; vy++) {
447       for (ex = 0; ex < edges[0]; ex++) {
448         PetscInt edge   = vy * edges[0] + ex;
449         PetscInt vertex = vy * (edges[0] + 1) + ex + numEdges;
450         PetscInt cone[2];
451 
452         cone[0] = vertex;
453         cone[1] = vertex + 1;
454         PetscCall(DMPlexSetCone(dm, edge, cone));
455         if (vy == edges[1]) {
456           PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
457           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
458           if (ex == edges[0] - 1) {
459             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
460             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerTop));
461           }
462         } else if (vy == 0) {
463           PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
464           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
465           if (ex == edges[0] - 1) {
466             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
467             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom));
468           }
469         }
470       }
471     }
472   }
473   PetscCall(DMPlexSymmetrize(dm));
474   PetscCall(DMPlexStratify(dm));
475   /* Build coordinates */
476   PetscCall(DMSetCoordinateDim(dm, 2));
477   PetscCall(DMGetCoordinateSection(dm, &coordSection));
478   PetscCall(PetscSectionSetNumFields(coordSection, 1));
479   PetscCall(PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices));
480   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 2));
481   for (v = numEdges; v < numEdges + numVertices; ++v) {
482     PetscCall(PetscSectionSetDof(coordSection, v, 2));
483     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 2));
484   }
485   PetscCall(PetscSectionSetUp(coordSection));
486   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
487   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
488   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
489   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
490   PetscCall(VecSetBlockSize(coordinates, 2));
491   PetscCall(VecSetType(coordinates, VECSTANDARD));
492   PetscCall(VecGetArray(coordinates, &coords));
493   for (vy = 0; vy <= edges[1]; ++vy) {
494     for (vx = 0; vx <= edges[0]; ++vx) {
495       coords[(vy * (edges[0] + 1) + vx) * 2 + 0] = lower[0] + ((upper[0] - lower[0]) / edges[0]) * vx;
496       coords[(vy * (edges[0] + 1) + vx) * 2 + 1] = lower[1] + ((upper[1] - lower[1]) / edges[1]) * vy;
497     }
498   }
499   PetscCall(VecRestoreArray(coordinates, &coords));
500   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
501   PetscCall(VecDestroy(&coordinates));
502   PetscFunctionReturn(0);
503 }
504 
505 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
506 {
507   PetscInt     vertices[3], numVertices;
508   PetscInt     numFaces       = 2 * faces[0] * faces[1] + 2 * faces[1] * faces[2] + 2 * faces[0] * faces[2];
509   PetscInt     markerTop      = 1;
510   PetscInt     markerBottom   = 1;
511   PetscInt     markerFront    = 1;
512   PetscInt     markerBack     = 1;
513   PetscInt     markerRight    = 1;
514   PetscInt     markerLeft     = 1;
515   PetscBool    markerSeparate = PETSC_FALSE;
516   Vec          coordinates;
517   PetscSection coordSection;
518   PetscScalar *coords;
519   PetscInt     coordSize;
520   PetscMPIInt  rank;
521   PetscInt     v, vx, vy, vz;
522   PetscInt     voffset, iface = 0, cone[4];
523 
524   PetscFunctionBegin;
525   PetscCheck(faces[0] >= 1 && faces[1] >= 1 && faces[2] >= 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
526   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
527   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
528   if (markerSeparate) {
529     markerBottom = 1;
530     markerTop    = 2;
531     markerFront  = 3;
532     markerBack   = 4;
533     markerRight  = 5;
534     markerLeft   = 6;
535   }
536   vertices[0] = faces[0] + 1;
537   vertices[1] = faces[1] + 1;
538   vertices[2] = faces[2] + 1;
539   numVertices = vertices[0] * vertices[1] * vertices[2];
540   if (rank == 0) {
541     PetscInt f;
542 
543     PetscCall(DMPlexSetChart(dm, 0, numFaces + numVertices));
544     for (f = 0; f < numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
545     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
546 
547     /* Side 0 (Top) */
548     for (vy = 0; vy < faces[1]; vy++) {
549       for (vx = 0; vx < faces[0]; vx++) {
550         voffset = numFaces + vertices[0] * vertices[1] * (vertices[2] - 1) + vy * vertices[0] + vx;
551         cone[0] = voffset;
552         cone[1] = voffset + 1;
553         cone[2] = voffset + vertices[0] + 1;
554         cone[3] = voffset + vertices[0];
555         PetscCall(DMPlexSetCone(dm, iface, cone));
556         PetscCall(DMSetLabelValue(dm, "marker", iface, markerTop));
557         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerTop));
558         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerTop));
559         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerTop));
560         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerTop));
561         iface++;
562       }
563     }
564 
565     /* Side 1 (Bottom) */
566     for (vy = 0; vy < faces[1]; vy++) {
567       for (vx = 0; vx < faces[0]; vx++) {
568         voffset = numFaces + vy * (faces[0] + 1) + vx;
569         cone[0] = voffset + 1;
570         cone[1] = voffset;
571         cone[2] = voffset + vertices[0];
572         cone[3] = voffset + vertices[0] + 1;
573         PetscCall(DMPlexSetCone(dm, iface, cone));
574         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBottom));
575         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBottom));
576         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBottom));
577         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerBottom));
578         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerBottom));
579         iface++;
580       }
581     }
582 
583     /* Side 2 (Front) */
584     for (vz = 0; vz < faces[2]; vz++) {
585       for (vx = 0; vx < faces[0]; vx++) {
586         voffset = numFaces + vz * vertices[0] * vertices[1] + vx;
587         cone[0] = voffset;
588         cone[1] = voffset + 1;
589         cone[2] = voffset + vertices[0] * vertices[1] + 1;
590         cone[3] = voffset + vertices[0] * vertices[1];
591         PetscCall(DMPlexSetCone(dm, iface, cone));
592         PetscCall(DMSetLabelValue(dm, "marker", iface, markerFront));
593         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerFront));
594         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerFront));
595         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerFront));
596         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerFront));
597         iface++;
598       }
599     }
600 
601     /* Side 3 (Back) */
602     for (vz = 0; vz < faces[2]; vz++) {
603       for (vx = 0; vx < faces[0]; vx++) {
604         voffset = numFaces + vz * vertices[0] * vertices[1] + vertices[0] * (vertices[1] - 1) + vx;
605         cone[0] = voffset + vertices[0] * vertices[1];
606         cone[1] = voffset + vertices[0] * vertices[1] + 1;
607         cone[2] = voffset + 1;
608         cone[3] = voffset;
609         PetscCall(DMPlexSetCone(dm, iface, cone));
610         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBack));
611         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBack));
612         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBack));
613         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerBack));
614         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerBack));
615         iface++;
616       }
617     }
618 
619     /* Side 4 (Left) */
620     for (vz = 0; vz < faces[2]; vz++) {
621       for (vy = 0; vy < faces[1]; vy++) {
622         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0];
623         cone[0] = voffset;
624         cone[1] = voffset + vertices[0] * vertices[1];
625         cone[2] = voffset + vertices[0] * vertices[1] + vertices[0];
626         cone[3] = voffset + vertices[0];
627         PetscCall(DMPlexSetCone(dm, iface, cone));
628         PetscCall(DMSetLabelValue(dm, "marker", iface, markerLeft));
629         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerLeft));
630         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerLeft));
631         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[1] + 0, markerLeft));
632         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerLeft));
633         iface++;
634       }
635     }
636 
637     /* Side 5 (Right) */
638     for (vz = 0; vz < faces[2]; vz++) {
639       for (vy = 0; vy < faces[1]; vy++) {
640         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0] + faces[0];
641         cone[0] = voffset + vertices[0] * vertices[1];
642         cone[1] = voffset;
643         cone[2] = voffset + vertices[0];
644         cone[3] = voffset + vertices[0] * vertices[1] + vertices[0];
645         PetscCall(DMPlexSetCone(dm, iface, cone));
646         PetscCall(DMSetLabelValue(dm, "marker", iface, markerRight));
647         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerRight));
648         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerRight));
649         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerRight));
650         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerRight));
651         iface++;
652       }
653     }
654   }
655   PetscCall(DMPlexSymmetrize(dm));
656   PetscCall(DMPlexStratify(dm));
657   /* Build coordinates */
658   PetscCall(DMSetCoordinateDim(dm, 3));
659   PetscCall(DMGetCoordinateSection(dm, &coordSection));
660   PetscCall(PetscSectionSetNumFields(coordSection, 1));
661   PetscCall(PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices));
662   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 3));
663   for (v = numFaces; v < numFaces + numVertices; ++v) {
664     PetscCall(PetscSectionSetDof(coordSection, v, 3));
665     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 3));
666   }
667   PetscCall(PetscSectionSetUp(coordSection));
668   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
669   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
670   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
671   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
672   PetscCall(VecSetBlockSize(coordinates, 3));
673   PetscCall(VecSetType(coordinates, VECSTANDARD));
674   PetscCall(VecGetArray(coordinates, &coords));
675   for (vz = 0; vz <= faces[2]; ++vz) {
676     for (vy = 0; vy <= faces[1]; ++vy) {
677       for (vx = 0; vx <= faces[0]; ++vx) {
678         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 0] = lower[0] + ((upper[0] - lower[0]) / faces[0]) * vx;
679         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 1] = lower[1] + ((upper[1] - lower[1]) / faces[1]) * vy;
680         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 2] = lower[2] + ((upper[2] - lower[2]) / faces[2]) * vz;
681       }
682     }
683   }
684   PetscCall(VecRestoreArray(coordinates, &coords));
685   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
686   PetscCall(VecDestroy(&coordinates));
687   PetscFunctionReturn(0);
688 }
689 
690 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate)
691 {
692   PetscFunctionBegin;
693   PetscValidLogicalCollectiveInt(dm, dim, 2);
694   PetscCall(DMSetDimension(dm, dim - 1));
695   PetscCall(DMSetCoordinateDim(dm, dim));
696   switch (dim) {
697   case 1:
698     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(dm, lower, upper, faces));
699     break;
700   case 2:
701     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(dm, lower, upper, faces));
702     break;
703   case 3:
704     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(dm, lower, upper, faces));
705     break;
706   default:
707     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Dimension not supported: %" PetscInt_FMT, dim);
708   }
709   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
710   PetscFunctionReturn(0);
711 }
712 
713 /*@C
714   DMPlexCreateBoxSurfaceMesh - Creates a mesh on the surface of the tensor product of unit intervals (box) using tensor cells (hexahedra).
715 
716   Collective
717 
718   Input Parameters:
719 + comm        - The communicator for the DM object
720 . dim         - The spatial dimension of the box, so the resulting mesh is has dimension dim-1
721 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
722 . lower       - The lower left corner, or NULL for (0, 0, 0)
723 . upper       - The upper right corner, or NULL for (1, 1, 1)
724 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
725 
726   Output Parameter:
727 . dm  - The DM object
728 
729   Level: beginner
730 
731 .seealso: `DMSetFromOptions()`, `DMPlexCreateBoxMesh()`, `DMPlexCreateFromFile()`, `DMSetType()`, `DMCreate()`
732 @*/
733 PetscErrorCode DMPlexCreateBoxSurfaceMesh(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate, DM *dm)
734 {
735   PetscInt  fac[3] = {1, 1, 1};
736   PetscReal low[3] = {0, 0, 0};
737   PetscReal upp[3] = {1, 1, 1};
738 
739   PetscFunctionBegin;
740   PetscCall(DMCreate(comm, dm));
741   PetscCall(DMSetType(*dm, DMPLEX));
742   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(*dm, dim, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, interpolate));
743   PetscFunctionReturn(0);
744 }
745 
746 static PetscErrorCode DMPlexCreateLineMesh_Internal(DM dm, PetscInt segments, PetscReal lower, PetscReal upper, DMBoundaryType bd)
747 {
748   PetscInt     i, fStart, fEnd, numCells = 0, numVerts = 0;
749   PetscInt     numPoints[2], *coneSize, *cones, *coneOrientations;
750   PetscScalar *vertexCoords;
751   PetscReal    L, maxCell;
752   PetscBool    markerSeparate = PETSC_FALSE;
753   PetscInt     markerLeft = 1, faceMarkerLeft = 1;
754   PetscInt     markerRight = 1, faceMarkerRight = 2;
755   PetscBool    wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
756   PetscMPIInt  rank;
757 
758   PetscFunctionBegin;
759   PetscValidPointer(dm, 1);
760 
761   PetscCall(DMSetDimension(dm, 1));
762   PetscCall(DMCreateLabel(dm, "marker"));
763   PetscCall(DMCreateLabel(dm, "Face Sets"));
764 
765   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
766   if (rank == 0) numCells = segments;
767   if (rank == 0) numVerts = segments + (wrap ? 0 : 1);
768 
769   numPoints[0] = numVerts;
770   numPoints[1] = numCells;
771   PetscCall(PetscMalloc4(numCells + numVerts, &coneSize, numCells * 2, &cones, numCells + numVerts, &coneOrientations, numVerts, &vertexCoords));
772   PetscCall(PetscArrayzero(coneOrientations, numCells + numVerts));
773   for (i = 0; i < numCells; ++i) coneSize[i] = 2;
774   for (i = 0; i < numVerts; ++i) coneSize[numCells + i] = 0;
775   for (i = 0; i < numCells; ++i) {
776     cones[2 * i]     = numCells + i % numVerts;
777     cones[2 * i + 1] = numCells + (i + 1) % numVerts;
778   }
779   for (i = 0; i < numVerts; ++i) vertexCoords[i] = lower + (upper - lower) * ((PetscReal)i / (PetscReal)numCells);
780   PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
781   PetscCall(PetscFree4(coneSize, cones, coneOrientations, vertexCoords));
782 
783   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
784   if (markerSeparate) {
785     markerLeft  = faceMarkerLeft;
786     markerRight = faceMarkerRight;
787   }
788   if (!wrap && rank == 0) {
789     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
790     PetscCall(DMSetLabelValue(dm, "marker", fStart, markerLeft));
791     PetscCall(DMSetLabelValue(dm, "marker", fEnd - 1, markerRight));
792     PetscCall(DMSetLabelValue(dm, "Face Sets", fStart, faceMarkerLeft));
793     PetscCall(DMSetLabelValue(dm, "Face Sets", fEnd - 1, faceMarkerRight));
794   }
795   if (wrap) {
796     L       = upper - lower;
797     maxCell = (PetscReal)1.1 * (L / (PetscReal)PetscMax(1, segments));
798     PetscCall(DMSetPeriodicity(dm, &maxCell, &lower, &L));
799   }
800   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
801   PetscFunctionReturn(0);
802 }
803 
804 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
805 {
806   DM      boundary, vol;
807   DMLabel bdlabel;
808 
809   PetscFunctionBegin;
810   PetscValidPointer(dm, 1);
811   for (PetscInt i = 0; i < dim; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
812   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &boundary));
813   PetscCall(DMSetType(boundary, DMPLEX));
814   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(boundary, dim, faces, lower, upper, PETSC_FALSE));
815   PetscCall(DMPlexGenerate(boundary, NULL, interpolate, &vol));
816   PetscCall(DMGetLabel(vol, "marker", &bdlabel));
817   if (bdlabel) PetscCall(DMPlexLabelComplete(vol, bdlabel));
818   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_FALSE, vol));
819   PetscCall(DMPlexReplace_Internal(dm, &vol));
820   PetscCall(DMDestroy(&boundary));
821   PetscFunctionReturn(0);
822 }
823 
824 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
825 {
826   DMLabel     cutLabel  = NULL;
827   PetscInt    markerTop = 1, faceMarkerTop = 1;
828   PetscInt    markerBottom = 1, faceMarkerBottom = 1;
829   PetscInt    markerFront = 1, faceMarkerFront = 1;
830   PetscInt    markerBack = 1, faceMarkerBack = 1;
831   PetscInt    markerRight = 1, faceMarkerRight = 1;
832   PetscInt    markerLeft = 1, faceMarkerLeft = 1;
833   PetscInt    dim;
834   PetscBool   markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
835   PetscMPIInt rank;
836 
837   PetscFunctionBegin;
838   PetscCall(DMGetDimension(dm, &dim));
839   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
840   PetscCall(DMCreateLabel(dm, "marker"));
841   PetscCall(DMCreateLabel(dm, "Face Sets"));
842   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
843   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST || bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST || bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
844     if (cutMarker) {
845       PetscCall(DMCreateLabel(dm, "periodic_cut"));
846       PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
847     }
848   }
849   switch (dim) {
850   case 2:
851     faceMarkerTop    = 3;
852     faceMarkerBottom = 1;
853     faceMarkerRight  = 2;
854     faceMarkerLeft   = 4;
855     break;
856   case 3:
857     faceMarkerBottom = 1;
858     faceMarkerTop    = 2;
859     faceMarkerFront  = 3;
860     faceMarkerBack   = 4;
861     faceMarkerRight  = 5;
862     faceMarkerLeft   = 6;
863     break;
864   default:
865     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
866   }
867   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
868   if (markerSeparate) {
869     markerBottom = faceMarkerBottom;
870     markerTop    = faceMarkerTop;
871     markerFront  = faceMarkerFront;
872     markerBack   = faceMarkerBack;
873     markerRight  = faceMarkerRight;
874     markerLeft   = faceMarkerLeft;
875   }
876   {
877     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
878     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
879     const PetscInt numZEdges    = rank == 0 ? edges[2] : 0;
880     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0] + 1) : 0;
881     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1] + 1) : 0;
882     const PetscInt numZVertices = rank == 0 ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2] + 1) : 0;
883     const PetscInt numCells     = numXEdges * numYEdges * numZEdges;
884     const PetscInt numXFaces    = numYEdges * numZEdges;
885     const PetscInt numYFaces    = numXEdges * numZEdges;
886     const PetscInt numZFaces    = numXEdges * numYEdges;
887     const PetscInt numTotXFaces = numXVertices * numXFaces;
888     const PetscInt numTotYFaces = numYVertices * numYFaces;
889     const PetscInt numTotZFaces = numZVertices * numZFaces;
890     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
891     const PetscInt numTotXEdges = numXEdges * numYVertices * numZVertices;
892     const PetscInt numTotYEdges = numYEdges * numXVertices * numZVertices;
893     const PetscInt numTotZEdges = numZEdges * numXVertices * numYVertices;
894     const PetscInt numVertices  = numXVertices * numYVertices * numZVertices;
895     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
896     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
897     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
898     const PetscInt firstYFace   = firstXFace + numTotXFaces;
899     const PetscInt firstZFace   = firstYFace + numTotYFaces;
900     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
901     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
902     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
903     Vec            coordinates;
904     PetscSection   coordSection;
905     PetscScalar   *coords;
906     PetscInt       coordSize;
907     PetscInt       v, vx, vy, vz;
908     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
909 
910     PetscCall(DMPlexSetChart(dm, 0, numCells + numFaces + numEdges + numVertices));
911     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
912     for (f = firstXFace; f < firstXFace + numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
913     for (e = firstXEdge; e < firstXEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
914     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
915     /* Build cells */
916     for (fz = 0; fz < numZEdges; ++fz) {
917       for (fy = 0; fy < numYEdges; ++fy) {
918         for (fx = 0; fx < numXEdges; ++fx) {
919           PetscInt cell  = (fz * numYEdges + fy) * numXEdges + fx;
920           PetscInt faceB = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
921           PetscInt faceT = firstZFace + (fy * numXEdges + fx) * numZVertices + ((fz + 1) % numZVertices);
922           PetscInt faceF = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
923           PetscInt faceK = firstYFace + (fz * numXEdges + fx) * numYVertices + ((fy + 1) % numYVertices);
924           PetscInt faceL = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
925           PetscInt faceR = firstXFace + (fz * numYEdges + fy) * numXVertices + ((fx + 1) % numXVertices);
926           /* B,  T,  F,  K,  R,  L */
927           PetscInt ornt[6] = {-2, 0, 0, -3, 0, -2}; /* ??? */
928           PetscInt cone[6];
929 
930           /* no boundary twisting in 3D */
931           cone[0] = faceB;
932           cone[1] = faceT;
933           cone[2] = faceF;
934           cone[3] = faceK;
935           cone[4] = faceR;
936           cone[5] = faceL;
937           PetscCall(DMPlexSetCone(dm, cell, cone));
938           PetscCall(DMPlexSetConeOrientation(dm, cell, ornt));
939           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
940           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
941           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
942         }
943       }
944     }
945     /* Build x faces */
946     for (fz = 0; fz < numZEdges; ++fz) {
947       for (fy = 0; fy < numYEdges; ++fy) {
948         for (fx = 0; fx < numXVertices; ++fx) {
949           PetscInt face    = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
950           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
951           PetscInt edgeR   = firstZEdge + (((fy + 1) % numYVertices) * numXVertices + fx) * numZEdges + fz;
952           PetscInt edgeB   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
953           PetscInt edgeT   = firstYEdge + (((fz + 1) % numZVertices) * numXVertices + fx) * numYEdges + fy;
954           PetscInt ornt[4] = {0, 0, -1, -1};
955           PetscInt cone[4];
956 
957           if (dim == 3) {
958             /* markers */
959             if (bdX != DM_BOUNDARY_PERIODIC) {
960               if (fx == numXVertices - 1) {
961                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight));
962                 PetscCall(DMSetLabelValue(dm, "marker", face, markerRight));
963               } else if (fx == 0) {
964                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft));
965                 PetscCall(DMSetLabelValue(dm, "marker", face, markerLeft));
966               }
967             }
968           }
969           cone[0] = edgeB;
970           cone[1] = edgeR;
971           cone[2] = edgeT;
972           cone[3] = edgeL;
973           PetscCall(DMPlexSetCone(dm, face, cone));
974           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
975         }
976       }
977     }
978     /* Build y faces */
979     for (fz = 0; fz < numZEdges; ++fz) {
980       for (fx = 0; fx < numXEdges; ++fx) {
981         for (fy = 0; fy < numYVertices; ++fy) {
982           PetscInt face    = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
983           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
984           PetscInt edgeR   = firstZEdge + (fy * numXVertices + ((fx + 1) % numXVertices)) * numZEdges + fz;
985           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
986           PetscInt edgeT   = firstXEdge + (((fz + 1) % numZVertices) * numYVertices + fy) * numXEdges + fx;
987           PetscInt ornt[4] = {0, 0, -1, -1};
988           PetscInt cone[4];
989 
990           if (dim == 3) {
991             /* markers */
992             if (bdY != DM_BOUNDARY_PERIODIC) {
993               if (fy == numYVertices - 1) {
994                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack));
995                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBack));
996               } else if (fy == 0) {
997                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront));
998                 PetscCall(DMSetLabelValue(dm, "marker", face, markerFront));
999               }
1000             }
1001           }
1002           cone[0] = edgeB;
1003           cone[1] = edgeR;
1004           cone[2] = edgeT;
1005           cone[3] = edgeL;
1006           PetscCall(DMPlexSetCone(dm, face, cone));
1007           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1008         }
1009       }
1010     }
1011     /* Build z faces */
1012     for (fy = 0; fy < numYEdges; ++fy) {
1013       for (fx = 0; fx < numXEdges; ++fx) {
1014         for (fz = 0; fz < numZVertices; fz++) {
1015           PetscInt face    = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
1016           PetscInt edgeL   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
1017           PetscInt edgeR   = firstYEdge + (fz * numXVertices + ((fx + 1) % numXVertices)) * numYEdges + fy;
1018           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1019           PetscInt edgeT   = firstXEdge + (fz * numYVertices + ((fy + 1) % numYVertices)) * numXEdges + fx;
1020           PetscInt ornt[4] = {0, 0, -1, -1};
1021           PetscInt cone[4];
1022 
1023           if (dim == 2) {
1024             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges - 1) {
1025               edgeR += numYEdges - 1 - 2 * fy;
1026               ornt[1] = -1;
1027             }
1028             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges - 1) {
1029               edgeT += numXEdges - 1 - 2 * fx;
1030               ornt[2] = 0;
1031             }
1032             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1033             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1034           } else {
1035             /* markers */
1036             if (bdZ != DM_BOUNDARY_PERIODIC) {
1037               if (fz == numZVertices - 1) {
1038                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop));
1039                 PetscCall(DMSetLabelValue(dm, "marker", face, markerTop));
1040               } else if (fz == 0) {
1041                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom));
1042                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBottom));
1043               }
1044             }
1045           }
1046           cone[0] = edgeB;
1047           cone[1] = edgeR;
1048           cone[2] = edgeT;
1049           cone[3] = edgeL;
1050           PetscCall(DMPlexSetCone(dm, face, cone));
1051           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1052         }
1053       }
1054     }
1055     /* Build Z edges*/
1056     for (vy = 0; vy < numYVertices; vy++) {
1057       for (vx = 0; vx < numXVertices; vx++) {
1058         for (ez = 0; ez < numZEdges; ez++) {
1059           const PetscInt edge    = firstZEdge + (vy * numXVertices + vx) * numZEdges + ez;
1060           const PetscInt vertexB = firstVertex + (ez * numYVertices + vy) * numXVertices + vx;
1061           const PetscInt vertexT = firstVertex + (((ez + 1) % numZVertices) * numYVertices + vy) * numXVertices + vx;
1062           PetscInt       cone[2];
1063 
1064           cone[0] = vertexB;
1065           cone[1] = vertexT;
1066           PetscCall(DMPlexSetCone(dm, edge, cone));
1067           if (dim == 3) {
1068             if (bdX != DM_BOUNDARY_PERIODIC) {
1069               if (vx == numXVertices - 1) {
1070                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1071                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1072                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1073               } else if (vx == 0) {
1074                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1075                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1076                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1077               }
1078             }
1079             if (bdY != DM_BOUNDARY_PERIODIC) {
1080               if (vy == numYVertices - 1) {
1081                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1082                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1083                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1084               } else if (vy == 0) {
1085                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1086                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1087                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1088               }
1089             }
1090           }
1091         }
1092       }
1093     }
1094     /* Build Y edges*/
1095     for (vz = 0; vz < numZVertices; vz++) {
1096       for (vx = 0; vx < numXVertices; vx++) {
1097         for (ey = 0; ey < numYEdges; ey++) {
1098           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges - 1) ? (numXVertices - vx - 1) : (vz * numYVertices + ((ey + 1) % numYVertices)) * numXVertices + vx;
1099           const PetscInt edge    = firstYEdge + (vz * numXVertices + vx) * numYEdges + ey;
1100           const PetscInt vertexF = firstVertex + (vz * numYVertices + ey) * numXVertices + vx;
1101           const PetscInt vertexK = firstVertex + nextv;
1102           PetscInt       cone[2];
1103 
1104           cone[0] = vertexF;
1105           cone[1] = vertexK;
1106           PetscCall(DMPlexSetCone(dm, edge, cone));
1107           if (dim == 2) {
1108             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1109               if (vx == numXVertices - 1) {
1110                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight));
1111                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1112                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1113                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1114               } else if (vx == 0) {
1115                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft));
1116                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1117                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1118                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1119               }
1120             } else {
1121               if (vx == 0 && cutLabel) {
1122                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1123                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1124                 if (ey == numYEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1125               }
1126             }
1127           } else {
1128             if (bdX != DM_BOUNDARY_PERIODIC) {
1129               if (vx == numXVertices - 1) {
1130                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1131                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1132                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1133               } else if (vx == 0) {
1134                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1135                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1136                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1137               }
1138             }
1139             if (bdZ != DM_BOUNDARY_PERIODIC) {
1140               if (vz == numZVertices - 1) {
1141                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1142                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1143                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1144               } else if (vz == 0) {
1145                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1146                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1147                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1148               }
1149             }
1150           }
1151         }
1152       }
1153     }
1154     /* Build X edges*/
1155     for (vz = 0; vz < numZVertices; vz++) {
1156       for (vy = 0; vy < numYVertices; vy++) {
1157         for (ex = 0; ex < numXEdges; ex++) {
1158           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges - 1) ? (numYVertices - vy - 1) * numXVertices : (vz * numYVertices + vy) * numXVertices + (ex + 1) % numXVertices;
1159           const PetscInt edge    = firstXEdge + (vz * numYVertices + vy) * numXEdges + ex;
1160           const PetscInt vertexL = firstVertex + (vz * numYVertices + vy) * numXVertices + ex;
1161           const PetscInt vertexR = firstVertex + nextv;
1162           PetscInt       cone[2];
1163 
1164           cone[0] = vertexL;
1165           cone[1] = vertexR;
1166           PetscCall(DMPlexSetCone(dm, edge, cone));
1167           if (dim == 2) {
1168             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1169               if (vy == numYVertices - 1) {
1170                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop));
1171                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1172                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1173                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1174               } else if (vy == 0) {
1175                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom));
1176                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1177                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1178                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1179               }
1180             } else {
1181               if (vy == 0 && cutLabel) {
1182                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1183                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1184                 if (ex == numXEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1185               }
1186             }
1187           } else {
1188             if (bdY != DM_BOUNDARY_PERIODIC) {
1189               if (vy == numYVertices - 1) {
1190                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1191                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1192                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1193               } else if (vy == 0) {
1194                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1195                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1196                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1197               }
1198             }
1199             if (bdZ != DM_BOUNDARY_PERIODIC) {
1200               if (vz == numZVertices - 1) {
1201                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1202                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1203                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1204               } else if (vz == 0) {
1205                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1206                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1207                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1208               }
1209             }
1210           }
1211         }
1212       }
1213     }
1214     PetscCall(DMPlexSymmetrize(dm));
1215     PetscCall(DMPlexStratify(dm));
1216     /* Build coordinates */
1217     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1218     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1219     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1220     PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVertices));
1221     for (v = firstVertex; v < firstVertex + numVertices; ++v) {
1222       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1223       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1224     }
1225     PetscCall(PetscSectionSetUp(coordSection));
1226     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1227     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1228     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1229     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1230     PetscCall(VecSetBlockSize(coordinates, dim));
1231     PetscCall(VecSetType(coordinates, VECSTANDARD));
1232     PetscCall(VecGetArray(coordinates, &coords));
1233     for (vz = 0; vz < numZVertices; ++vz) {
1234       for (vy = 0; vy < numYVertices; ++vy) {
1235         for (vx = 0; vx < numXVertices; ++vx) {
1236           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * vx;
1237           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * vy;
1238           if (dim == 3) coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 2] = lower[2] + ((upper[2] - lower[2]) / numZEdges) * vz;
1239         }
1240       }
1241     }
1242     PetscCall(VecRestoreArray(coordinates, &coords));
1243     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1244     PetscCall(VecDestroy(&coordinates));
1245   }
1246   PetscFunctionReturn(0);
1247 }
1248 
1249 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1250 {
1251   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1252   PetscInt       fac[3] = {0, 0, 0}, d;
1253 
1254   PetscFunctionBegin;
1255   PetscValidPointer(dm, 1);
1256   PetscValidLogicalCollectiveInt(dm, dim, 2);
1257   PetscCall(DMSetDimension(dm, dim));
1258   for (d = 0; d < dim; ++d) {
1259     fac[d] = faces[d];
1260     bdt[d] = periodicity[d];
1261   }
1262   PetscCall(DMPlexCreateCubeMesh_Internal(dm, lower, upper, fac, bdt[0], bdt[1], bdt[2]));
1263   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST || periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST || (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
1264     PetscReal L[3]       = {-1., -1., 0.};
1265     PetscReal maxCell[3] = {-1., -1., 0.};
1266 
1267     for (d = 0; d < dim; ++d) {
1268       if (periodicity[d] != DM_BOUNDARY_NONE) {
1269         L[d]       = upper[d] - lower[d];
1270         maxCell[d] = 1.1 * (L[d] / PetscMax(1, faces[d]));
1271       }
1272     }
1273     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1274   }
1275   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
1276   PetscFunctionReturn(0);
1277 }
1278 
1279 static PetscErrorCode DMPlexCreateBoxMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1280 {
1281   PetscFunctionBegin;
1282   if (dim == 1) PetscCall(DMPlexCreateLineMesh_Internal(dm, faces[0], lower[0], upper[0], periodicity[0]));
1283   else if (simplex) PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1284   else PetscCall(DMPlexCreateBoxMesh_Tensor_Internal(dm, dim, faces, lower, upper, periodicity));
1285   if (!interpolate && dim > 1 && !simplex) {
1286     DM udm;
1287 
1288     PetscCall(DMPlexUninterpolate(dm, &udm));
1289     PetscCall(DMPlexCopyCoordinates(dm, udm));
1290     PetscCall(DMPlexReplace_Internal(dm, &udm));
1291   }
1292   PetscFunctionReturn(0);
1293 }
1294 
1295 /*@C
1296   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
1297 
1298   Collective
1299 
1300   Input Parameters:
1301 + comm        - The communicator for the DM object
1302 . dim         - The spatial dimension
1303 . simplex     - PETSC_TRUE for simplices, PETSC_FALSE for tensor cells
1304 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1305 . lower       - The lower left corner, or NULL for (0, 0, 0)
1306 . upper       - The upper right corner, or NULL for (1, 1, 1)
1307 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
1308 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1309 
1310   Output Parameter:
1311 . dm  - The DM object
1312 
1313   Note: If you want to customize this mesh using options, you just need to
1314 $  DMCreate(comm, &dm);
1315 $  DMSetType(dm, DMPLEX);
1316 $  DMSetFromOptions(dm);
1317 and use the options on the DMSetFromOptions() page.
1318 
1319   Here is the numbering returned for 2 faces in each direction for tensor cells:
1320 $ 10---17---11---18----12
1321 $  |         |         |
1322 $  |         |         |
1323 $ 20    2   22    3    24
1324 $  |         |         |
1325 $  |         |         |
1326 $  7---15----8---16----9
1327 $  |         |         |
1328 $  |         |         |
1329 $ 19    0   21    1   23
1330 $  |         |         |
1331 $  |         |         |
1332 $  4---13----5---14----6
1333 
1334 and for simplicial cells
1335 
1336 $ 14----8---15----9----16
1337 $  |\     5  |\      7 |
1338 $  | \       | \       |
1339 $ 13   2    14    3    15
1340 $  | 4   \   | 6   \   |
1341 $  |       \ |       \ |
1342 $ 11----6---12----7----13
1343 $  |\        |\        |
1344 $  | \    1  | \     3 |
1345 $ 10   0    11    1    12
1346 $  | 0   \   | 2   \   |
1347 $  |       \ |       \ |
1348 $  8----4----9----5----10
1349 
1350   Level: beginner
1351 
1352 .seealso: `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
1353 @*/
1354 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
1355 {
1356   PetscInt       fac[3] = {1, 1, 1};
1357   PetscReal      low[3] = {0, 0, 0};
1358   PetscReal      upp[3] = {1, 1, 1};
1359   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1360 
1361   PetscFunctionBegin;
1362   PetscCall(DMCreate(comm, dm));
1363   PetscCall(DMSetType(*dm, DMPLEX));
1364   PetscCall(DMPlexCreateBoxMesh_Internal(*dm, dim, simplex, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt, interpolate));
1365   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1366   PetscFunctionReturn(0);
1367 }
1368 
1369 static PetscErrorCode DMPlexCreateWedgeBoxMesh_Internal(DM dm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1370 {
1371   DM       bdm, vol;
1372   PetscInt i;
1373 
1374   PetscFunctionBegin;
1375   for (i = 0; i < 3; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity not yet supported");
1376   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &bdm));
1377   PetscCall(DMSetType(bdm, DMPLEX));
1378   PetscCall(DMSetDimension(bdm, 2));
1379   PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(bdm, 2, faces, lower, upper, periodicity, PETSC_TRUE));
1380   PetscCall(DMPlexExtrude(bdm, faces[2], upper[2] - lower[2], PETSC_TRUE, PETSC_FALSE, NULL, NULL, &vol));
1381   PetscCall(DMDestroy(&bdm));
1382   PetscCall(DMPlexReplace_Internal(dm, &vol));
1383   if (lower[2] != 0.0) {
1384     Vec          v;
1385     PetscScalar *x;
1386     PetscInt     cDim, n;
1387 
1388     PetscCall(DMGetCoordinatesLocal(dm, &v));
1389     PetscCall(VecGetBlockSize(v, &cDim));
1390     PetscCall(VecGetLocalSize(v, &n));
1391     PetscCall(VecGetArray(v, &x));
1392     x += cDim;
1393     for (i = 0; i < n; i += cDim) x[i] += lower[2];
1394     PetscCall(VecRestoreArray(v, &x));
1395     PetscCall(DMSetCoordinatesLocal(dm, v));
1396   }
1397   PetscFunctionReturn(0);
1398 }
1399 
1400 /*@
1401   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tesselating the (x,y) plane and extruding in the third direction using wedge cells.
1402 
1403   Collective
1404 
1405   Input Parameters:
1406 + comm        - The communicator for the DM object
1407 . faces       - Number of faces per dimension, or NULL for (1, 1, 1)
1408 . lower       - The lower left corner, or NULL for (0, 0, 0)
1409 . upper       - The upper right corner, or NULL for (1, 1, 1)
1410 . periodicity - The boundary type for the X,Y,Z direction, or NULL for DM_BOUNDARY_NONE
1411 . orderHeight - If PETSC_TRUE, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1412 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1413 
1414   Output Parameter:
1415 . dm  - The DM object
1416 
1417   Level: beginner
1418 
1419 .seealso: `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateWedgeCylinderMesh()`, `DMExtrude()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1420 @*/
1421 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool orderHeight, PetscBool interpolate, DM *dm)
1422 {
1423   PetscInt       fac[3] = {1, 1, 1};
1424   PetscReal      low[3] = {0, 0, 0};
1425   PetscReal      upp[3] = {1, 1, 1};
1426   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1427 
1428   PetscFunctionBegin;
1429   PetscCall(DMCreate(comm, dm));
1430   PetscCall(DMSetType(*dm, DMPLEX));
1431   PetscCall(DMPlexCreateWedgeBoxMesh_Internal(*dm, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt));
1432   if (!interpolate) {
1433     DM udm;
1434 
1435     PetscCall(DMPlexUninterpolate(*dm, &udm));
1436     PetscCall(DMPlexReplace_Internal(*dm, &udm));
1437   }
1438   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1439   PetscFunctionReturn(0);
1440 }
1441 
1442 /*@C
1443   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all DM options in the database.
1444 
1445   Logically Collective on dm
1446 
1447   Input Parameters:
1448 + dm - the DM context
1449 - prefix - the prefix to prepend to all option names
1450 
1451   Notes:
1452   A hyphen (-) must NOT be given at the beginning of the prefix name.
1453   The first character of all runtime options is AUTOMATICALLY the hyphen.
1454 
1455   Level: advanced
1456 
1457 .seealso: `SNESSetFromOptions()`
1458 @*/
1459 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
1460 {
1461   DM_Plex *mesh = (DM_Plex *)dm->data;
1462 
1463   PetscFunctionBegin;
1464   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1465   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, prefix));
1466   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)mesh->partitioner, prefix));
1467   PetscFunctionReturn(0);
1468 }
1469 
1470 /* Remap geometry to cylinder
1471    TODO: This only works for a single refinement, then it is broken
1472 
1473      Interior square: Linear interpolation is correct
1474      The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
1475      such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
1476 
1477        phi     = arctan(y/x)
1478        d_close = sqrt(1/8 + 1/4 sin^2(phi))
1479        d_far   = sqrt(1/2 + sin^2(phi))
1480 
1481      so we remap them using
1482 
1483        x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
1484        y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
1485 
1486      If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
1487 */
1488 static void snapToCylinder(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
1489 {
1490   const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1491   const PetscReal ds2 = 0.5 * dis;
1492 
1493   if ((PetscAbsScalar(u[0]) <= ds2) && (PetscAbsScalar(u[1]) <= ds2)) {
1494     f0[0] = u[0];
1495     f0[1] = u[1];
1496   } else {
1497     PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
1498 
1499     x    = PetscRealPart(u[0]);
1500     y    = PetscRealPart(u[1]);
1501     phi  = PetscAtan2Real(y, x);
1502     sinp = PetscSinReal(phi);
1503     cosp = PetscCosReal(phi);
1504     if ((PetscAbsReal(phi) > PETSC_PI / 4.0) && (PetscAbsReal(phi) < 3.0 * PETSC_PI / 4.0)) {
1505       dc = PetscAbsReal(ds2 / sinp);
1506       df = PetscAbsReal(dis / sinp);
1507       xc = ds2 * x / PetscAbsReal(y);
1508       yc = ds2 * PetscSignReal(y);
1509     } else {
1510       dc = PetscAbsReal(ds2 / cosp);
1511       df = PetscAbsReal(dis / cosp);
1512       xc = ds2 * PetscSignReal(x);
1513       yc = ds2 * y / PetscAbsReal(x);
1514     }
1515     f0[0] = xc + (u[0] - xc) * (1.0 - dc) / (df - dc);
1516     f0[1] = yc + (u[1] - yc) * (1.0 - dc) / (df - dc);
1517   }
1518   f0[2] = u[2];
1519 }
1520 
1521 static PetscErrorCode DMPlexCreateHexCylinderMesh_Internal(DM dm, DMBoundaryType periodicZ)
1522 {
1523   const PetscInt dim = 3;
1524   PetscInt       numCells, numVertices;
1525   PetscMPIInt    rank;
1526 
1527   PetscFunctionBegin;
1528   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1529   PetscCall(DMSetDimension(dm, dim));
1530   /* Create topology */
1531   {
1532     PetscInt cone[8], c;
1533 
1534     numCells    = rank == 0 ? 5 : 0;
1535     numVertices = rank == 0 ? 16 : 0;
1536     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1537       numCells *= 3;
1538       numVertices = rank == 0 ? 24 : 0;
1539     }
1540     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1541     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 8));
1542     PetscCall(DMSetUp(dm));
1543     if (rank == 0) {
1544       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1545         cone[0] = 15;
1546         cone[1] = 18;
1547         cone[2] = 17;
1548         cone[3] = 16;
1549         cone[4] = 31;
1550         cone[5] = 32;
1551         cone[6] = 33;
1552         cone[7] = 34;
1553         PetscCall(DMPlexSetCone(dm, 0, cone));
1554         cone[0] = 16;
1555         cone[1] = 17;
1556         cone[2] = 24;
1557         cone[3] = 23;
1558         cone[4] = 32;
1559         cone[5] = 36;
1560         cone[6] = 37;
1561         cone[7] = 33; /* 22 25 26 21 */
1562         PetscCall(DMPlexSetCone(dm, 1, cone));
1563         cone[0] = 18;
1564         cone[1] = 27;
1565         cone[2] = 24;
1566         cone[3] = 17;
1567         cone[4] = 34;
1568         cone[5] = 33;
1569         cone[6] = 37;
1570         cone[7] = 38;
1571         PetscCall(DMPlexSetCone(dm, 2, cone));
1572         cone[0] = 29;
1573         cone[1] = 27;
1574         cone[2] = 18;
1575         cone[3] = 15;
1576         cone[4] = 35;
1577         cone[5] = 31;
1578         cone[6] = 34;
1579         cone[7] = 38;
1580         PetscCall(DMPlexSetCone(dm, 3, cone));
1581         cone[0] = 29;
1582         cone[1] = 15;
1583         cone[2] = 16;
1584         cone[3] = 23;
1585         cone[4] = 35;
1586         cone[5] = 36;
1587         cone[6] = 32;
1588         cone[7] = 31;
1589         PetscCall(DMPlexSetCone(dm, 4, cone));
1590 
1591         cone[0] = 31;
1592         cone[1] = 34;
1593         cone[2] = 33;
1594         cone[3] = 32;
1595         cone[4] = 19;
1596         cone[5] = 22;
1597         cone[6] = 21;
1598         cone[7] = 20;
1599         PetscCall(DMPlexSetCone(dm, 5, cone));
1600         cone[0] = 32;
1601         cone[1] = 33;
1602         cone[2] = 37;
1603         cone[3] = 36;
1604         cone[4] = 22;
1605         cone[5] = 25;
1606         cone[6] = 26;
1607         cone[7] = 21;
1608         PetscCall(DMPlexSetCone(dm, 6, cone));
1609         cone[0] = 34;
1610         cone[1] = 38;
1611         cone[2] = 37;
1612         cone[3] = 33;
1613         cone[4] = 20;
1614         cone[5] = 21;
1615         cone[6] = 26;
1616         cone[7] = 28;
1617         PetscCall(DMPlexSetCone(dm, 7, cone));
1618         cone[0] = 35;
1619         cone[1] = 38;
1620         cone[2] = 34;
1621         cone[3] = 31;
1622         cone[4] = 30;
1623         cone[5] = 19;
1624         cone[6] = 20;
1625         cone[7] = 28;
1626         PetscCall(DMPlexSetCone(dm, 8, cone));
1627         cone[0] = 35;
1628         cone[1] = 31;
1629         cone[2] = 32;
1630         cone[3] = 36;
1631         cone[4] = 30;
1632         cone[5] = 25;
1633         cone[6] = 22;
1634         cone[7] = 19;
1635         PetscCall(DMPlexSetCone(dm, 9, cone));
1636 
1637         cone[0] = 19;
1638         cone[1] = 20;
1639         cone[2] = 21;
1640         cone[3] = 22;
1641         cone[4] = 15;
1642         cone[5] = 16;
1643         cone[6] = 17;
1644         cone[7] = 18;
1645         PetscCall(DMPlexSetCone(dm, 10, cone));
1646         cone[0] = 22;
1647         cone[1] = 21;
1648         cone[2] = 26;
1649         cone[3] = 25;
1650         cone[4] = 16;
1651         cone[5] = 23;
1652         cone[6] = 24;
1653         cone[7] = 17;
1654         PetscCall(DMPlexSetCone(dm, 11, cone));
1655         cone[0] = 20;
1656         cone[1] = 28;
1657         cone[2] = 26;
1658         cone[3] = 21;
1659         cone[4] = 18;
1660         cone[5] = 17;
1661         cone[6] = 24;
1662         cone[7] = 27;
1663         PetscCall(DMPlexSetCone(dm, 12, cone));
1664         cone[0] = 30;
1665         cone[1] = 28;
1666         cone[2] = 20;
1667         cone[3] = 19;
1668         cone[4] = 29;
1669         cone[5] = 15;
1670         cone[6] = 18;
1671         cone[7] = 27;
1672         PetscCall(DMPlexSetCone(dm, 13, cone));
1673         cone[0] = 30;
1674         cone[1] = 19;
1675         cone[2] = 22;
1676         cone[3] = 25;
1677         cone[4] = 29;
1678         cone[5] = 23;
1679         cone[6] = 16;
1680         cone[7] = 15;
1681         PetscCall(DMPlexSetCone(dm, 14, cone));
1682       } else {
1683         cone[0] = 5;
1684         cone[1] = 8;
1685         cone[2] = 7;
1686         cone[3] = 6;
1687         cone[4] = 9;
1688         cone[5] = 12;
1689         cone[6] = 11;
1690         cone[7] = 10;
1691         PetscCall(DMPlexSetCone(dm, 0, cone));
1692         cone[0] = 6;
1693         cone[1] = 7;
1694         cone[2] = 14;
1695         cone[3] = 13;
1696         cone[4] = 12;
1697         cone[5] = 15;
1698         cone[6] = 16;
1699         cone[7] = 11;
1700         PetscCall(DMPlexSetCone(dm, 1, cone));
1701         cone[0] = 8;
1702         cone[1] = 17;
1703         cone[2] = 14;
1704         cone[3] = 7;
1705         cone[4] = 10;
1706         cone[5] = 11;
1707         cone[6] = 16;
1708         cone[7] = 18;
1709         PetscCall(DMPlexSetCone(dm, 2, cone));
1710         cone[0] = 19;
1711         cone[1] = 17;
1712         cone[2] = 8;
1713         cone[3] = 5;
1714         cone[4] = 20;
1715         cone[5] = 9;
1716         cone[6] = 10;
1717         cone[7] = 18;
1718         PetscCall(DMPlexSetCone(dm, 3, cone));
1719         cone[0] = 19;
1720         cone[1] = 5;
1721         cone[2] = 6;
1722         cone[3] = 13;
1723         cone[4] = 20;
1724         cone[5] = 15;
1725         cone[6] = 12;
1726         cone[7] = 9;
1727         PetscCall(DMPlexSetCone(dm, 4, cone));
1728       }
1729     }
1730     PetscCall(DMPlexSymmetrize(dm));
1731     PetscCall(DMPlexStratify(dm));
1732   }
1733   /* Create cube geometry */
1734   {
1735     Vec             coordinates;
1736     PetscSection    coordSection;
1737     PetscScalar    *coords;
1738     PetscInt        coordSize, v;
1739     const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1740     const PetscReal ds2 = dis / 2.0;
1741 
1742     /* Build coordinates */
1743     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1744     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1745     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1746     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1747     for (v = numCells; v < numCells + numVertices; ++v) {
1748       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1749       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1750     }
1751     PetscCall(PetscSectionSetUp(coordSection));
1752     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1753     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1754     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1755     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1756     PetscCall(VecSetBlockSize(coordinates, dim));
1757     PetscCall(VecSetType(coordinates, VECSTANDARD));
1758     PetscCall(VecGetArray(coordinates, &coords));
1759     if (rank == 0) {
1760       coords[0 * dim + 0]  = -ds2;
1761       coords[0 * dim + 1]  = -ds2;
1762       coords[0 * dim + 2]  = 0.0;
1763       coords[1 * dim + 0]  = ds2;
1764       coords[1 * dim + 1]  = -ds2;
1765       coords[1 * dim + 2]  = 0.0;
1766       coords[2 * dim + 0]  = ds2;
1767       coords[2 * dim + 1]  = ds2;
1768       coords[2 * dim + 2]  = 0.0;
1769       coords[3 * dim + 0]  = -ds2;
1770       coords[3 * dim + 1]  = ds2;
1771       coords[3 * dim + 2]  = 0.0;
1772       coords[4 * dim + 0]  = -ds2;
1773       coords[4 * dim + 1]  = -ds2;
1774       coords[4 * dim + 2]  = 1.0;
1775       coords[5 * dim + 0]  = -ds2;
1776       coords[5 * dim + 1]  = ds2;
1777       coords[5 * dim + 2]  = 1.0;
1778       coords[6 * dim + 0]  = ds2;
1779       coords[6 * dim + 1]  = ds2;
1780       coords[6 * dim + 2]  = 1.0;
1781       coords[7 * dim + 0]  = ds2;
1782       coords[7 * dim + 1]  = -ds2;
1783       coords[7 * dim + 2]  = 1.0;
1784       coords[8 * dim + 0]  = dis;
1785       coords[8 * dim + 1]  = -dis;
1786       coords[8 * dim + 2]  = 0.0;
1787       coords[9 * dim + 0]  = dis;
1788       coords[9 * dim + 1]  = dis;
1789       coords[9 * dim + 2]  = 0.0;
1790       coords[10 * dim + 0] = dis;
1791       coords[10 * dim + 1] = -dis;
1792       coords[10 * dim + 2] = 1.0;
1793       coords[11 * dim + 0] = dis;
1794       coords[11 * dim + 1] = dis;
1795       coords[11 * dim + 2] = 1.0;
1796       coords[12 * dim + 0] = -dis;
1797       coords[12 * dim + 1] = dis;
1798       coords[12 * dim + 2] = 0.0;
1799       coords[13 * dim + 0] = -dis;
1800       coords[13 * dim + 1] = dis;
1801       coords[13 * dim + 2] = 1.0;
1802       coords[14 * dim + 0] = -dis;
1803       coords[14 * dim + 1] = -dis;
1804       coords[14 * dim + 2] = 0.0;
1805       coords[15 * dim + 0] = -dis;
1806       coords[15 * dim + 1] = -dis;
1807       coords[15 * dim + 2] = 1.0;
1808       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1809         /* 15 31 19 */ coords[16 * dim + 0] = -ds2;
1810         coords[16 * dim + 1]                = -ds2;
1811         coords[16 * dim + 2]                = 0.5;
1812         /* 16 32 22 */ coords[17 * dim + 0] = ds2;
1813         coords[17 * dim + 1]                = -ds2;
1814         coords[17 * dim + 2]                = 0.5;
1815         /* 17 33 21 */ coords[18 * dim + 0] = ds2;
1816         coords[18 * dim + 1]                = ds2;
1817         coords[18 * dim + 2]                = 0.5;
1818         /* 18 34 20 */ coords[19 * dim + 0] = -ds2;
1819         coords[19 * dim + 1]                = ds2;
1820         coords[19 * dim + 2]                = 0.5;
1821         /* 29 35 30 */ coords[20 * dim + 0] = -dis;
1822         coords[20 * dim + 1]                = -dis;
1823         coords[20 * dim + 2]                = 0.5;
1824         /* 23 36 25 */ coords[21 * dim + 0] = dis;
1825         coords[21 * dim + 1]                = -dis;
1826         coords[21 * dim + 2]                = 0.5;
1827         /* 24 37 26 */ coords[22 * dim + 0] = dis;
1828         coords[22 * dim + 1]                = dis;
1829         coords[22 * dim + 2]                = 0.5;
1830         /* 27 38 28 */ coords[23 * dim + 0] = -dis;
1831         coords[23 * dim + 1]                = dis;
1832         coords[23 * dim + 2]                = 0.5;
1833       }
1834     }
1835     PetscCall(VecRestoreArray(coordinates, &coords));
1836     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1837     PetscCall(VecDestroy(&coordinates));
1838   }
1839   /* Create periodicity */
1840   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
1841     PetscReal L[3]       = {-1., -1., 0.};
1842     PetscReal maxCell[3] = {-1., -1., 0.};
1843     PetscReal lower[3]   = {0.0, 0.0, 0.0};
1844     PetscReal upper[3]   = {1.0, 1.0, 1.5};
1845     PetscInt  numZCells  = 3;
1846 
1847     L[2]       = upper[2] - lower[2];
1848     maxCell[2] = 1.1 * (L[2] / numZCells);
1849     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1850   }
1851   {
1852     DM          cdm;
1853     PetscDS     cds;
1854     PetscScalar c[2] = {1.0, 1.0};
1855 
1856     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToCylinder));
1857     PetscCall(DMGetCoordinateDM(dm, &cdm));
1858     PetscCall(DMGetDS(cdm, &cds));
1859     PetscCall(PetscDSSetConstants(cds, 2, c));
1860   }
1861   /* Wait for coordinate creation before doing in-place modification */
1862   PetscCall(DMPlexInterpolateInPlace_Internal(dm));
1863   PetscFunctionReturn(0);
1864 }
1865 
1866 /*@
1867   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
1868 
1869   Collective
1870 
1871   Input Parameters:
1872 + comm      - The communicator for the DM object
1873 - periodicZ - The boundary type for the Z direction
1874 
1875   Output Parameter:
1876 . dm  - The DM object
1877 
1878   Note:
1879   Here is the output numbering looking from the bottom of the cylinder:
1880 $       17-----14
1881 $        |     |
1882 $        |  2  |
1883 $        |     |
1884 $ 17-----8-----7-----14
1885 $  |     |     |     |
1886 $  |  3  |  0  |  1  |
1887 $  |     |     |     |
1888 $ 19-----5-----6-----13
1889 $        |     |
1890 $        |  4  |
1891 $        |     |
1892 $       19-----13
1893 $
1894 $ and up through the top
1895 $
1896 $       18-----16
1897 $        |     |
1898 $        |  2  |
1899 $        |     |
1900 $ 18----10----11-----16
1901 $  |     |     |     |
1902 $  |  3  |  0  |  1  |
1903 $  |     |     |     |
1904 $ 20-----9----12-----15
1905 $        |     |
1906 $        |  4  |
1907 $        |     |
1908 $       20-----15
1909 
1910   Level: beginner
1911 
1912 .seealso: `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1913 @*/
1914 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, DMBoundaryType periodicZ, DM *dm)
1915 {
1916   PetscFunctionBegin;
1917   PetscValidPointer(dm, 3);
1918   PetscCall(DMCreate(comm, dm));
1919   PetscCall(DMSetType(*dm, DMPLEX));
1920   PetscCall(DMPlexCreateHexCylinderMesh_Internal(*dm, periodicZ));
1921   PetscFunctionReturn(0);
1922 }
1923 
1924 static PetscErrorCode DMPlexCreateWedgeCylinderMesh_Internal(DM dm, PetscInt n, PetscBool interpolate)
1925 {
1926   const PetscInt dim = 3;
1927   PetscInt       numCells, numVertices, v;
1928   PetscMPIInt    rank;
1929 
1930   PetscFunctionBegin;
1931   PetscCheck(n >= 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %" PetscInt_FMT " cannot be negative", n);
1932   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1933   PetscCall(DMSetDimension(dm, dim));
1934   /* Must create the celltype label here so that we do not automatically try to compute the types */
1935   PetscCall(DMCreateLabel(dm, "celltype"));
1936   /* Create topology */
1937   {
1938     PetscInt cone[6], c;
1939 
1940     numCells    = rank == 0 ? n : 0;
1941     numVertices = rank == 0 ? 2 * (n + 1) : 0;
1942     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1943     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
1944     PetscCall(DMSetUp(dm));
1945     for (c = 0; c < numCells; c++) {
1946       cone[0] = c + n * 1;
1947       cone[1] = (c + 1) % n + n * 1;
1948       cone[2] = 0 + 3 * n;
1949       cone[3] = c + n * 2;
1950       cone[4] = (c + 1) % n + n * 2;
1951       cone[5] = 1 + 3 * n;
1952       PetscCall(DMPlexSetCone(dm, c, cone));
1953       PetscCall(DMPlexSetCellType(dm, c, DM_POLYTOPE_TRI_PRISM_TENSOR));
1954     }
1955     PetscCall(DMPlexSymmetrize(dm));
1956     PetscCall(DMPlexStratify(dm));
1957   }
1958   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetCellType(dm, v, DM_POLYTOPE_POINT));
1959   /* Create cylinder geometry */
1960   {
1961     Vec          coordinates;
1962     PetscSection coordSection;
1963     PetscScalar *coords;
1964     PetscInt     coordSize, c;
1965 
1966     /* Build coordinates */
1967     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1968     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1969     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1970     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1971     for (v = numCells; v < numCells + numVertices; ++v) {
1972       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1973       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1974     }
1975     PetscCall(PetscSectionSetUp(coordSection));
1976     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1977     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1978     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1979     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1980     PetscCall(VecSetBlockSize(coordinates, dim));
1981     PetscCall(VecSetType(coordinates, VECSTANDARD));
1982     PetscCall(VecGetArray(coordinates, &coords));
1983     for (c = 0; c < numCells; c++) {
1984       coords[(c + 0 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
1985       coords[(c + 0 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
1986       coords[(c + 0 * n) * dim + 2] = 1.0;
1987       coords[(c + 1 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
1988       coords[(c + 1 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
1989       coords[(c + 1 * n) * dim + 2] = 0.0;
1990     }
1991     if (rank == 0) {
1992       coords[(2 * n + 0) * dim + 0] = 0.0;
1993       coords[(2 * n + 0) * dim + 1] = 0.0;
1994       coords[(2 * n + 0) * dim + 2] = 1.0;
1995       coords[(2 * n + 1) * dim + 0] = 0.0;
1996       coords[(2 * n + 1) * dim + 1] = 0.0;
1997       coords[(2 * n + 1) * dim + 2] = 0.0;
1998     }
1999     PetscCall(VecRestoreArray(coordinates, &coords));
2000     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2001     PetscCall(VecDestroy(&coordinates));
2002   }
2003   /* Interpolate */
2004   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2005   PetscFunctionReturn(0);
2006 }
2007 
2008 /*@
2009   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
2010 
2011   Collective
2012 
2013   Input Parameters:
2014 + comm - The communicator for the DM object
2015 . n    - The number of wedges around the origin
2016 - interpolate - Create edges and faces
2017 
2018   Output Parameter:
2019 . dm  - The DM object
2020 
2021   Level: beginner
2022 
2023 .seealso: `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2024 @*/
2025 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
2026 {
2027   PetscFunctionBegin;
2028   PetscValidPointer(dm, 4);
2029   PetscCall(DMCreate(comm, dm));
2030   PetscCall(DMSetType(*dm, DMPLEX));
2031   PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(*dm, n, interpolate));
2032   PetscFunctionReturn(0);
2033 }
2034 
2035 static inline PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2036 {
2037   PetscReal prod = 0.0;
2038   PetscInt  i;
2039   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
2040   return PetscSqrtReal(prod);
2041 }
2042 static inline PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2043 {
2044   PetscReal prod = 0.0;
2045   PetscInt  i;
2046   for (i = 0; i < dim; ++i) prod += x[i] * y[i];
2047   return prod;
2048 }
2049 
2050 /* The first constant is the sphere radius */
2051 static void snapToSphere(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
2052 {
2053   PetscReal r     = PetscRealPart(constants[0]);
2054   PetscReal norm2 = 0.0, fac;
2055   PetscInt  n     = uOff[1] - uOff[0], d;
2056 
2057   for (d = 0; d < n; ++d) norm2 += PetscSqr(PetscRealPart(u[d]));
2058   fac = r / PetscSqrtReal(norm2);
2059   for (d = 0; d < n; ++d) f0[d] = u[d] * fac;
2060 }
2061 
2062 static PetscErrorCode DMPlexCreateSphereMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, PetscReal R)
2063 {
2064   const PetscInt embedDim = dim + 1;
2065   PetscSection   coordSection;
2066   Vec            coordinates;
2067   PetscScalar   *coords;
2068   PetscReal     *coordsIn;
2069   PetscInt       numCells, numEdges, numVerts = 0, firstVertex = 0, v, firstEdge, coordSize, d, c, e;
2070   PetscMPIInt    rank;
2071 
2072   PetscFunctionBegin;
2073   PetscValidLogicalCollectiveBool(dm, simplex, 3);
2074   PetscCall(DMSetDimension(dm, dim));
2075   PetscCall(DMSetCoordinateDim(dm, dim + 1));
2076   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2077   switch (dim) {
2078   case 2:
2079     if (simplex) {
2080       const PetscReal radius    = PetscSqrtReal(1 + PETSC_PHI * PETSC_PHI) / (1.0 + PETSC_PHI);
2081       const PetscReal edgeLen   = 2.0 / (1.0 + PETSC_PHI) * (R / radius);
2082       const PetscInt  degree    = 5;
2083       PetscReal       vertex[3] = {0.0, 1.0 / (1.0 + PETSC_PHI), PETSC_PHI / (1.0 + PETSC_PHI)};
2084       PetscInt        s[3]      = {1, 1, 1};
2085       PetscInt        cone[3];
2086       PetscInt       *graph, p, i, j, k;
2087 
2088       vertex[0] *= R / radius;
2089       vertex[1] *= R / radius;
2090       vertex[2] *= R / radius;
2091       numCells    = rank == 0 ? 20 : 0;
2092       numVerts    = rank == 0 ? 12 : 0;
2093       firstVertex = numCells;
2094       /* Use icosahedron, which for a R-sphere has coordinates which are all cyclic permutations of
2095 
2096            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
2097 
2098          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2099          length is then given by 2/(1+\phi) = 2 * 0.38197 = 0.76393.
2100       */
2101       /* Construct vertices */
2102       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2103       if (rank == 0) {
2104         for (p = 0, i = 0; p < embedDim; ++p) {
2105           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2106             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2107               for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertex[(d + p) % embedDim];
2108               ++i;
2109             }
2110           }
2111         }
2112       }
2113       /* Construct graph */
2114       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2115       for (i = 0; i < numVerts; ++i) {
2116         for (j = 0, k = 0; j < numVerts; ++j) {
2117           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2118             graph[i * numVerts + j] = 1;
2119             ++k;
2120           }
2121         }
2122         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid icosahedron, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2123       }
2124       /* Build Topology */
2125       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2126       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2127       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2128       /* Cells */
2129       for (i = 0, c = 0; i < numVerts; ++i) {
2130         for (j = 0; j < i; ++j) {
2131           for (k = 0; k < j; ++k) {
2132             if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i]) {
2133               cone[0] = firstVertex + i;
2134               cone[1] = firstVertex + j;
2135               cone[2] = firstVertex + k;
2136               /* Check orientation */
2137               {
2138                 const PetscInt epsilon[3][3][3] = {
2139                   {{0, 0, 0},  {0, 0, 1},  {0, -1, 0}},
2140                   {{0, 0, -1}, {0, 0, 0},  {1, 0, 0} },
2141                   {{0, 1, 0},  {-1, 0, 0}, {0, 0, 0} }
2142                 };
2143                 PetscReal normal[3];
2144                 PetscInt  e, f;
2145 
2146                 for (d = 0; d < embedDim; ++d) {
2147                   normal[d] = 0.0;
2148                   for (e = 0; e < embedDim; ++e) {
2149                     for (f = 0; f < embedDim; ++f) normal[d] += epsilon[d][e][f] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]);
2150                   }
2151                 }
2152                 if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2153                   PetscInt tmp = cone[1];
2154                   cone[1]      = cone[2];
2155                   cone[2]      = tmp;
2156                 }
2157               }
2158               PetscCall(DMPlexSetCone(dm, c++, cone));
2159             }
2160           }
2161         }
2162       }
2163       PetscCall(DMPlexSymmetrize(dm));
2164       PetscCall(DMPlexStratify(dm));
2165       PetscCall(PetscFree(graph));
2166     } else {
2167       /*
2168         12-21--13
2169          |     |
2170         25  4  24
2171          |     |
2172   12-25--9-16--8-24--13
2173    |     |     |     |
2174   23  5 17  0 15  3  22
2175    |     |     |     |
2176   10-20--6-14--7-19--11
2177          |     |
2178         20  1  19
2179          |     |
2180         10-18--11
2181          |     |
2182         23  2  22
2183          |     |
2184         12-21--13
2185        */
2186       PetscInt cone[4], ornt[4];
2187 
2188       numCells    = rank == 0 ? 6 : 0;
2189       numEdges    = rank == 0 ? 12 : 0;
2190       numVerts    = rank == 0 ? 8 : 0;
2191       firstVertex = numCells;
2192       firstEdge   = numCells + numVerts;
2193       /* Build Topology */
2194       PetscCall(DMPlexSetChart(dm, 0, numCells + numEdges + numVerts));
2195       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 4));
2196       for (e = firstEdge; e < firstEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
2197       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2198       if (rank == 0) {
2199         /* Cell 0 */
2200         cone[0] = 14;
2201         cone[1] = 15;
2202         cone[2] = 16;
2203         cone[3] = 17;
2204         PetscCall(DMPlexSetCone(dm, 0, cone));
2205         ornt[0] = 0;
2206         ornt[1] = 0;
2207         ornt[2] = 0;
2208         ornt[3] = 0;
2209         PetscCall(DMPlexSetConeOrientation(dm, 0, ornt));
2210         /* Cell 1 */
2211         cone[0] = 18;
2212         cone[1] = 19;
2213         cone[2] = 14;
2214         cone[3] = 20;
2215         PetscCall(DMPlexSetCone(dm, 1, cone));
2216         ornt[0] = 0;
2217         ornt[1] = 0;
2218         ornt[2] = -1;
2219         ornt[3] = 0;
2220         PetscCall(DMPlexSetConeOrientation(dm, 1, ornt));
2221         /* Cell 2 */
2222         cone[0] = 21;
2223         cone[1] = 22;
2224         cone[2] = 18;
2225         cone[3] = 23;
2226         PetscCall(DMPlexSetCone(dm, 2, cone));
2227         ornt[0] = 0;
2228         ornt[1] = 0;
2229         ornt[2] = -1;
2230         ornt[3] = 0;
2231         PetscCall(DMPlexSetConeOrientation(dm, 2, ornt));
2232         /* Cell 3 */
2233         cone[0] = 19;
2234         cone[1] = 22;
2235         cone[2] = 24;
2236         cone[3] = 15;
2237         PetscCall(DMPlexSetCone(dm, 3, cone));
2238         ornt[0] = -1;
2239         ornt[1] = -1;
2240         ornt[2] = 0;
2241         ornt[3] = -1;
2242         PetscCall(DMPlexSetConeOrientation(dm, 3, ornt));
2243         /* Cell 4 */
2244         cone[0] = 16;
2245         cone[1] = 24;
2246         cone[2] = 21;
2247         cone[3] = 25;
2248         PetscCall(DMPlexSetCone(dm, 4, cone));
2249         ornt[0] = -1;
2250         ornt[1] = -1;
2251         ornt[2] = -1;
2252         ornt[3] = 0;
2253         PetscCall(DMPlexSetConeOrientation(dm, 4, ornt));
2254         /* Cell 5 */
2255         cone[0] = 20;
2256         cone[1] = 17;
2257         cone[2] = 25;
2258         cone[3] = 23;
2259         PetscCall(DMPlexSetCone(dm, 5, cone));
2260         ornt[0] = -1;
2261         ornt[1] = -1;
2262         ornt[2] = -1;
2263         ornt[3] = -1;
2264         PetscCall(DMPlexSetConeOrientation(dm, 5, ornt));
2265         /* Edges */
2266         cone[0] = 6;
2267         cone[1] = 7;
2268         PetscCall(DMPlexSetCone(dm, 14, cone));
2269         cone[0] = 7;
2270         cone[1] = 8;
2271         PetscCall(DMPlexSetCone(dm, 15, cone));
2272         cone[0] = 8;
2273         cone[1] = 9;
2274         PetscCall(DMPlexSetCone(dm, 16, cone));
2275         cone[0] = 9;
2276         cone[1] = 6;
2277         PetscCall(DMPlexSetCone(dm, 17, cone));
2278         cone[0] = 10;
2279         cone[1] = 11;
2280         PetscCall(DMPlexSetCone(dm, 18, cone));
2281         cone[0] = 11;
2282         cone[1] = 7;
2283         PetscCall(DMPlexSetCone(dm, 19, cone));
2284         cone[0] = 6;
2285         cone[1] = 10;
2286         PetscCall(DMPlexSetCone(dm, 20, cone));
2287         cone[0] = 12;
2288         cone[1] = 13;
2289         PetscCall(DMPlexSetCone(dm, 21, cone));
2290         cone[0] = 13;
2291         cone[1] = 11;
2292         PetscCall(DMPlexSetCone(dm, 22, cone));
2293         cone[0] = 10;
2294         cone[1] = 12;
2295         PetscCall(DMPlexSetCone(dm, 23, cone));
2296         cone[0] = 13;
2297         cone[1] = 8;
2298         PetscCall(DMPlexSetCone(dm, 24, cone));
2299         cone[0] = 12;
2300         cone[1] = 9;
2301         PetscCall(DMPlexSetCone(dm, 25, cone));
2302       }
2303       PetscCall(DMPlexSymmetrize(dm));
2304       PetscCall(DMPlexStratify(dm));
2305       /* Build coordinates */
2306       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2307       if (rank == 0) {
2308         coordsIn[0 * embedDim + 0] = -R;
2309         coordsIn[0 * embedDim + 1] = R;
2310         coordsIn[0 * embedDim + 2] = -R;
2311         coordsIn[1 * embedDim + 0] = R;
2312         coordsIn[1 * embedDim + 1] = R;
2313         coordsIn[1 * embedDim + 2] = -R;
2314         coordsIn[2 * embedDim + 0] = R;
2315         coordsIn[2 * embedDim + 1] = -R;
2316         coordsIn[2 * embedDim + 2] = -R;
2317         coordsIn[3 * embedDim + 0] = -R;
2318         coordsIn[3 * embedDim + 1] = -R;
2319         coordsIn[3 * embedDim + 2] = -R;
2320         coordsIn[4 * embedDim + 0] = -R;
2321         coordsIn[4 * embedDim + 1] = R;
2322         coordsIn[4 * embedDim + 2] = R;
2323         coordsIn[5 * embedDim + 0] = R;
2324         coordsIn[5 * embedDim + 1] = R;
2325         coordsIn[5 * embedDim + 2] = R;
2326         coordsIn[6 * embedDim + 0] = -R;
2327         coordsIn[6 * embedDim + 1] = -R;
2328         coordsIn[6 * embedDim + 2] = R;
2329         coordsIn[7 * embedDim + 0] = R;
2330         coordsIn[7 * embedDim + 1] = -R;
2331         coordsIn[7 * embedDim + 2] = R;
2332       }
2333     }
2334     break;
2335   case 3:
2336     if (simplex) {
2337       const PetscReal edgeLen         = 1.0 / PETSC_PHI;
2338       PetscReal       vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
2339       PetscReal       vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
2340       PetscReal       vertexC[4]      = {0.5, 0.5 * PETSC_PHI, 0.5 / PETSC_PHI, 0.0};
2341       const PetscInt  degree          = 12;
2342       PetscInt        s[4]            = {1, 1, 1};
2343       PetscInt        evenPerm[12][4] = {
2344         {0, 1, 2, 3},
2345         {0, 2, 3, 1},
2346         {0, 3, 1, 2},
2347         {1, 0, 3, 2},
2348         {1, 2, 0, 3},
2349         {1, 3, 2, 0},
2350         {2, 0, 1, 3},
2351         {2, 1, 3, 0},
2352         {2, 3, 0, 1},
2353         {3, 0, 2, 1},
2354         {3, 1, 0, 2},
2355         {3, 2, 1, 0}
2356       };
2357       PetscInt  cone[4];
2358       PetscInt *graph, p, i, j, k, l;
2359 
2360       vertexA[0] *= R;
2361       vertexA[1] *= R;
2362       vertexA[2] *= R;
2363       vertexA[3] *= R;
2364       vertexB[0] *= R;
2365       vertexB[1] *= R;
2366       vertexB[2] *= R;
2367       vertexB[3] *= R;
2368       vertexC[0] *= R;
2369       vertexC[1] *= R;
2370       vertexC[2] *= R;
2371       vertexC[3] *= R;
2372       numCells    = rank == 0 ? 600 : 0;
2373       numVerts    = rank == 0 ? 120 : 0;
2374       firstVertex = numCells;
2375       /* Use the 600-cell, which for a unit sphere has coordinates which are
2376 
2377            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
2378                (\pm 1,    0,       0,      0)  all cyclic permutations   8
2379            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
2380 
2381          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2382          length is then given by 1/\phi = 0.61803.
2383 
2384          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
2385          http://mathworld.wolfram.com/600-Cell.html
2386       */
2387       /* Construct vertices */
2388       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2389       i = 0;
2390       if (rank == 0) {
2391         for (s[0] = -1; s[0] < 2; s[0] += 2) {
2392           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2393             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2394               for (s[3] = -1; s[3] < 2; s[3] += 2) {
2395                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[d] * vertexA[d];
2396                 ++i;
2397               }
2398             }
2399           }
2400         }
2401         for (p = 0; p < embedDim; ++p) {
2402           s[1] = s[2] = s[3] = 1;
2403           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2404             for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertexB[(d + p) % embedDim];
2405             ++i;
2406           }
2407         }
2408         for (p = 0; p < 12; ++p) {
2409           s[3] = 1;
2410           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2411             for (s[1] = -1; s[1] < 2; s[1] += 2) {
2412               for (s[2] = -1; s[2] < 2; s[2] += 2) {
2413                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[evenPerm[p][d]] * vertexC[evenPerm[p][d]];
2414                 ++i;
2415               }
2416             }
2417           }
2418         }
2419       }
2420       PetscCheck(i == numVerts, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertices %" PetscInt_FMT " != %" PetscInt_FMT, i, numVerts);
2421       /* Construct graph */
2422       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2423       for (i = 0; i < numVerts; ++i) {
2424         for (j = 0, k = 0; j < numVerts; ++j) {
2425           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2426             graph[i * numVerts + j] = 1;
2427             ++k;
2428           }
2429         }
2430         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2431       }
2432       /* Build Topology */
2433       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2434       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2435       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2436       /* Cells */
2437       if (rank == 0) {
2438         for (i = 0, c = 0; i < numVerts; ++i) {
2439           for (j = 0; j < i; ++j) {
2440             for (k = 0; k < j; ++k) {
2441               for (l = 0; l < k; ++l) {
2442                 if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i] && graph[l * numVerts + i] && graph[l * numVerts + j] && graph[l * numVerts + k]) {
2443                   cone[0] = firstVertex + i;
2444                   cone[1] = firstVertex + j;
2445                   cone[2] = firstVertex + k;
2446                   cone[3] = firstVertex + l;
2447                   /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
2448                   {
2449                     const PetscInt epsilon[4][4][4][4] = {
2450                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, -1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 0, 0}, {0, 1, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 1, 0}, {0, -1, 0, 0}, {0, 0, 0, 0}}},
2451 
2452                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}}, {{0, 0, -1, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}}},
2453 
2454                       {{{0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, 0, 0}, {0, -1, 0, 0}}, {{0, 0, 0, -1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 1, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}},
2455 
2456                       {{{0, 0, 0, 0}, {0, 0, -1, 0}, {0, 1, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 1, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}}, {{0, -1, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}} }
2457                     };
2458                     PetscReal normal[4];
2459                     PetscInt  e, f, g;
2460 
2461                     for (d = 0; d < embedDim; ++d) {
2462                       normal[d] = 0.0;
2463                       for (e = 0; e < embedDim; ++e) {
2464                         for (f = 0; f < embedDim; ++f) {
2465                           for (g = 0; g < embedDim; ++g) {
2466                             normal[d] += epsilon[d][e][f][g] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]) * (coordsIn[l * embedDim + f] - coordsIn[i * embedDim + f]);
2467                           }
2468                         }
2469                       }
2470                     }
2471                     if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2472                       PetscInt tmp = cone[1];
2473                       cone[1]      = cone[2];
2474                       cone[2]      = tmp;
2475                     }
2476                   }
2477                   PetscCall(DMPlexSetCone(dm, c++, cone));
2478                 }
2479               }
2480             }
2481           }
2482         }
2483       }
2484       PetscCall(DMPlexSymmetrize(dm));
2485       PetscCall(DMPlexStratify(dm));
2486       PetscCall(PetscFree(graph));
2487     }
2488     break;
2489   default:
2490     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unsupported dimension for sphere: %" PetscInt_FMT, dim);
2491   }
2492   /* Create coordinates */
2493   PetscCall(DMGetCoordinateSection(dm, &coordSection));
2494   PetscCall(PetscSectionSetNumFields(coordSection, 1));
2495   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, embedDim));
2496   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVerts));
2497   for (v = firstVertex; v < firstVertex + numVerts; ++v) {
2498     PetscCall(PetscSectionSetDof(coordSection, v, embedDim));
2499     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, embedDim));
2500   }
2501   PetscCall(PetscSectionSetUp(coordSection));
2502   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2503   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2504   PetscCall(VecSetBlockSize(coordinates, embedDim));
2505   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2506   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2507   PetscCall(VecSetType(coordinates, VECSTANDARD));
2508   PetscCall(VecGetArray(coordinates, &coords));
2509   for (v = 0; v < numVerts; ++v)
2510     for (d = 0; d < embedDim; ++d) coords[v * embedDim + d] = coordsIn[v * embedDim + d];
2511   PetscCall(VecRestoreArray(coordinates, &coords));
2512   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2513   PetscCall(VecDestroy(&coordinates));
2514   PetscCall(PetscFree(coordsIn));
2515   {
2516     DM          cdm;
2517     PetscDS     cds;
2518     PetscScalar c = R;
2519 
2520     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToSphere));
2521     PetscCall(DMGetCoordinateDM(dm, &cdm));
2522     PetscCall(DMGetDS(cdm, &cds));
2523     PetscCall(PetscDSSetConstants(cds, 1, &c));
2524   }
2525   /* Wait for coordinate creation before doing in-place modification */
2526   if (simplex) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2527   PetscFunctionReturn(0);
2528 }
2529 
2530 typedef void (*TPSEvaluateFunc)(const PetscReal[], PetscReal *, PetscReal[], PetscReal (*)[3]);
2531 
2532 /*
2533  The Schwarz P implicit surface is
2534 
2535      f(x) = cos(x0) + cos(x1) + cos(x2) = 0
2536 */
2537 static void TPSEvaluate_SchwarzP(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2538 {
2539   PetscReal c[3] = {PetscCosReal(y[0] * PETSC_PI), PetscCosReal(y[1] * PETSC_PI), PetscCosReal(y[2] * PETSC_PI)};
2540   PetscReal g[3] = {-PetscSinReal(y[0] * PETSC_PI), -PetscSinReal(y[1] * PETSC_PI), -PetscSinReal(y[2] * PETSC_PI)};
2541   f[0]           = c[0] + c[1] + c[2];
2542   for (PetscInt i = 0; i < 3; i++) {
2543     grad[i] = PETSC_PI * g[i];
2544     for (PetscInt j = 0; j < 3; j++) hess[i][j] = (i == j) ? -PetscSqr(PETSC_PI) * c[i] : 0.;
2545   }
2546 }
2547 
2548 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2549 static PetscErrorCode TPSExtrudeNormalFunc_SchwarzP(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2550 {
2551   for (PetscInt i = 0; i < 3; i++) u[i] = -PETSC_PI * PetscSinReal(x[i] * PETSC_PI);
2552   return 0;
2553 }
2554 
2555 /*
2556  The Gyroid implicit surface is
2557 
2558  f(x,y,z) = sin(pi * x) * cos (pi * (y + 1/2))  + sin(pi * (y + 1/2)) * cos(pi * (z + 1/4)) + sin(pi * (z + 1/4)) * cos(pi * x)
2559 
2560 */
2561 static void TPSEvaluate_Gyroid(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2562 {
2563   PetscReal s[3] = {PetscSinReal(PETSC_PI * y[0]), PetscSinReal(PETSC_PI * (y[1] + .5)), PetscSinReal(PETSC_PI * (y[2] + .25))};
2564   PetscReal c[3] = {PetscCosReal(PETSC_PI * y[0]), PetscCosReal(PETSC_PI * (y[1] + .5)), PetscCosReal(PETSC_PI * (y[2] + .25))};
2565   f[0]           = s[0] * c[1] + s[1] * c[2] + s[2] * c[0];
2566   grad[0]        = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2567   grad[1]        = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2568   grad[2]        = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2569   hess[0][0]     = -PetscSqr(PETSC_PI) * (s[0] * c[1] + s[2] * c[0]);
2570   hess[0][1]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2571   hess[0][2]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2572   hess[1][0]     = -PetscSqr(PETSC_PI) * (s[1] * c[2] + s[0] * c[1]);
2573   hess[1][1]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2574   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2575   hess[2][0]     = -PetscSqr(PETSC_PI) * (s[2] * c[0] + s[1] * c[2]);
2576   hess[2][1]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2577   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2578 }
2579 
2580 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2581 static PetscErrorCode TPSExtrudeNormalFunc_Gyroid(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2582 {
2583   PetscReal s[3] = {PetscSinReal(PETSC_PI * x[0]), PetscSinReal(PETSC_PI * (x[1] + .5)), PetscSinReal(PETSC_PI * (x[2] + .25))};
2584   PetscReal c[3] = {PetscCosReal(PETSC_PI * x[0]), PetscCosReal(PETSC_PI * (x[1] + .5)), PetscCosReal(PETSC_PI * (x[2] + .25))};
2585   u[0]           = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2586   u[1]           = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2587   u[2]           = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2588   return 0;
2589 }
2590 
2591 /*
2592    We wish to solve
2593 
2594          min_y || y - x ||^2  subject to f(y) = 0
2595 
2596    Let g(y) = grad(f).  The minimization problem is equivalent to asking to satisfy
2597    f(y) = 0 and (y-x) is parallel to g(y).  We do this by using Householder QR to obtain a basis for the
2598    tangent space and ask for both components in the tangent space to be zero.
2599 
2600    Take g to be a column vector and compute the "full QR" factorization Q R = g,
2601    where Q = I - 2 n n^T is a symmetric orthogonal matrix.
2602    The first column of Q is parallel to g so the remaining two columns span the null space.
2603    Let Qn = Q[:,1:] be those remaining columns.  Then Qn Qn^T is an orthogonal projector into the tangent space.
2604    Since Q is symmetric, this is equivalent to multipyling by Q and taking the last two entries.
2605    In total, we have a system of 3 equations in 3 unknowns:
2606 
2607      f(y) = 0                       1 equation
2608      Qn^T (y - x) = 0               2 equations
2609 
2610    Here, we compute the residual and Jacobian of this system.
2611 */
2612 static void TPSNearestPointResJac(TPSEvaluateFunc feval, const PetscScalar x[], const PetscScalar y[], PetscScalar res[], PetscScalar J[])
2613 {
2614   PetscReal yreal[3] = {PetscRealPart(y[0]), PetscRealPart(y[1]), PetscRealPart(y[2])};
2615   PetscReal d[3]     = {PetscRealPart(y[0] - x[0]), PetscRealPart(y[1] - x[1]), PetscRealPart(y[2] - x[2])};
2616   PetscReal f, grad[3], n[3], norm, norm_y[3], nd, nd_y[3], sign;
2617   PetscReal n_y[3][3] = {
2618     {0, 0, 0},
2619     {0, 0, 0},
2620     {0, 0, 0}
2621   };
2622 
2623   feval(yreal, &f, grad, n_y);
2624 
2625   for (PetscInt i = 0; i < 3; i++) n[i] = grad[i];
2626   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2627   for (PetscInt i = 0; i < 3; i++) norm_y[i] = 1. / norm * n[i] * n_y[i][i];
2628 
2629   // Define the Householder reflector
2630   sign = n[0] >= 0 ? 1. : -1.;
2631   n[0] += norm * sign;
2632   for (PetscInt i = 0; i < 3; i++) n_y[0][i] += norm_y[i] * sign;
2633 
2634   norm      = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2635   norm_y[0] = 1. / norm * (n[0] * n_y[0][0]);
2636   norm_y[1] = 1. / norm * (n[0] * n_y[0][1] + n[1] * n_y[1][1]);
2637   norm_y[2] = 1. / norm * (n[0] * n_y[0][2] + n[2] * n_y[2][2]);
2638 
2639   for (PetscInt i = 0; i < 3; i++) {
2640     n[i] /= norm;
2641     for (PetscInt j = 0; j < 3; j++) {
2642       // note that n[i] is n_old[i]/norm when executing the code below
2643       n_y[i][j] = n_y[i][j] / norm - n[i] / norm * norm_y[j];
2644     }
2645   }
2646 
2647   nd = n[0] * d[0] + n[1] * d[1] + n[2] * d[2];
2648   for (PetscInt i = 0; i < 3; i++) nd_y[i] = n[i] + n_y[0][i] * d[0] + n_y[1][i] * d[1] + n_y[2][i] * d[2];
2649 
2650   res[0] = f;
2651   res[1] = d[1] - 2 * n[1] * nd;
2652   res[2] = d[2] - 2 * n[2] * nd;
2653   // J[j][i] is J_{ij} (column major)
2654   for (PetscInt j = 0; j < 3; j++) {
2655     J[0 + j * 3] = grad[j];
2656     J[1 + j * 3] = (j == 1) * 1. - 2 * (n_y[1][j] * nd + n[1] * nd_y[j]);
2657     J[2 + j * 3] = (j == 2) * 1. - 2 * (n_y[2][j] * nd + n[2] * nd_y[j]);
2658   }
2659 }
2660 
2661 /*
2662    Project x to the nearest point on the implicit surface using Newton's method.
2663 */
2664 static PetscErrorCode TPSNearestPoint(TPSEvaluateFunc feval, PetscScalar x[])
2665 {
2666   PetscScalar y[3] = {x[0], x[1], x[2]}; // Initial guess
2667 
2668   PetscFunctionBegin;
2669   for (PetscInt iter = 0; iter < 10; iter++) {
2670     PetscScalar res[3], J[9];
2671     PetscReal   resnorm;
2672     TPSNearestPointResJac(feval, x, y, res, J);
2673     resnorm = PetscSqrtReal(PetscSqr(PetscRealPart(res[0])) + PetscSqr(PetscRealPart(res[1])) + PetscSqr(PetscRealPart(res[2])));
2674     if (0) { // Turn on this monitor if you need to confirm quadratic convergence
2675       PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%" PetscInt_FMT "] res [%g %g %g]\n", iter, (double)PetscRealPart(res[0]), (double)PetscRealPart(res[1]), (double)PetscRealPart(res[2])));
2676     }
2677     if (resnorm < PETSC_SMALL) break;
2678 
2679     // Take the Newton step
2680     PetscCall(PetscKernel_A_gets_inverse_A_3(J, 0., PETSC_FALSE, NULL));
2681     PetscKernel_v_gets_v_minus_A_times_w_3(y, J, res);
2682   }
2683   for (PetscInt i = 0; i < 3; i++) x[i] = y[i];
2684   PetscFunctionReturn(0);
2685 }
2686 
2687 const char *const DMPlexTPSTypes[] = {"SCHWARZ_P", "GYROID", "DMPlexTPSType", "DMPLEX_TPS_", NULL};
2688 
2689 static PetscErrorCode DMPlexCreateTPSMesh_Internal(DM dm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness)
2690 {
2691   PetscMPIInt rank;
2692   PetscInt    topoDim = 2, spaceDim = 3, numFaces = 0, numVertices = 0, numEdges = 0;
2693   PetscInt(*edges)[2] = NULL, *edgeSets = NULL;
2694   PetscInt            *cells_flat = NULL;
2695   PetscReal           *vtxCoords  = NULL;
2696   TPSEvaluateFunc      evalFunc   = NULL;
2697   PetscSimplePointFunc normalFunc = NULL;
2698   DMLabel              label;
2699 
2700   PetscFunctionBegin;
2701   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2702   PetscCheck((layers != 0) ^ (thickness == 0.), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Layers %" PetscInt_FMT " must be nonzero iff thickness %g is nonzero", layers, (double)thickness);
2703   switch (tpstype) {
2704   case DMPLEX_TPS_SCHWARZ_P:
2705     PetscCheck(!periodic || (periodic[0] == DM_BOUNDARY_NONE && periodic[1] == DM_BOUNDARY_NONE && periodic[2] == DM_BOUNDARY_NONE), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Schwarz P does not support periodic meshes");
2706     if (rank == 0) {
2707       PetscInt(*cells)[6][4][4] = NULL; // [junction, junction-face, cell, conn]
2708       PetscInt  Njunctions = 0, Ncuts = 0, Npipes[3], vcount;
2709       PetscReal L = 1;
2710 
2711       Npipes[0]   = (extent[0] + 1) * extent[1] * extent[2];
2712       Npipes[1]   = extent[0] * (extent[1] + 1) * extent[2];
2713       Npipes[2]   = extent[0] * extent[1] * (extent[2] + 1);
2714       Njunctions  = extent[0] * extent[1] * extent[2];
2715       Ncuts       = 2 * (extent[0] * extent[1] + extent[1] * extent[2] + extent[2] * extent[0]);
2716       numVertices = 4 * (Npipes[0] + Npipes[1] + Npipes[2]) + 8 * Njunctions;
2717       PetscCall(PetscMalloc1(3 * numVertices, &vtxCoords));
2718       PetscCall(PetscMalloc1(Njunctions, &cells));
2719       PetscCall(PetscMalloc1(Ncuts * 4, &edges));
2720       PetscCall(PetscMalloc1(Ncuts * 4, &edgeSets));
2721       // x-normal pipes
2722       vcount = 0;
2723       for (PetscInt i = 0; i < extent[0] + 1; i++) {
2724         for (PetscInt j = 0; j < extent[1]; j++) {
2725           for (PetscInt k = 0; k < extent[2]; k++) {
2726             for (PetscInt l = 0; l < 4; l++) {
2727               vtxCoords[vcount++] = (2 * i - 1) * L;
2728               vtxCoords[vcount++] = 2 * j * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2729               vtxCoords[vcount++] = 2 * k * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2730             }
2731           }
2732         }
2733       }
2734       // y-normal pipes
2735       for (PetscInt i = 0; i < extent[0]; i++) {
2736         for (PetscInt j = 0; j < extent[1] + 1; j++) {
2737           for (PetscInt k = 0; k < extent[2]; k++) {
2738             for (PetscInt l = 0; l < 4; l++) {
2739               vtxCoords[vcount++] = 2 * i * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2740               vtxCoords[vcount++] = (2 * j - 1) * L;
2741               vtxCoords[vcount++] = 2 * k * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2742             }
2743           }
2744         }
2745       }
2746       // z-normal pipes
2747       for (PetscInt i = 0; i < extent[0]; i++) {
2748         for (PetscInt j = 0; j < extent[1]; j++) {
2749           for (PetscInt k = 0; k < extent[2] + 1; k++) {
2750             for (PetscInt l = 0; l < 4; l++) {
2751               vtxCoords[vcount++] = 2 * i * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2752               vtxCoords[vcount++] = 2 * j * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2753               vtxCoords[vcount++] = (2 * k - 1) * L;
2754             }
2755           }
2756         }
2757       }
2758       // junctions
2759       for (PetscInt i = 0; i < extent[0]; i++) {
2760         for (PetscInt j = 0; j < extent[1]; j++) {
2761           for (PetscInt k = 0; k < extent[2]; k++) {
2762             const PetscInt J = (i * extent[1] + j) * extent[2] + k, Jvoff = (Npipes[0] + Npipes[1] + Npipes[2]) * 4 + J * 8;
2763             PetscCheck(vcount / 3 == Jvoff, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected vertex count");
2764             for (PetscInt ii = 0; ii < 2; ii++) {
2765               for (PetscInt jj = 0; jj < 2; jj++) {
2766                 for (PetscInt kk = 0; kk < 2; kk++) {
2767                   double Ls           = (1 - sqrt(2) / 4) * L;
2768                   vtxCoords[vcount++] = 2 * i * L + (2 * ii - 1) * Ls;
2769                   vtxCoords[vcount++] = 2 * j * L + (2 * jj - 1) * Ls;
2770                   vtxCoords[vcount++] = 2 * k * L + (2 * kk - 1) * Ls;
2771                 }
2772               }
2773             }
2774             const PetscInt jfaces[3][2][4] = {
2775               {{3, 1, 0, 2}, {7, 5, 4, 6}}, // x-aligned
2776               {{5, 4, 0, 1}, {7, 6, 2, 3}}, // y-aligned
2777               {{6, 2, 0, 4}, {7, 3, 1, 5}}  // z-aligned
2778             };
2779             const PetscInt pipe_lo[3] = {// vertex numbers of pipes
2780                                          ((i * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + Npipes[0] + Npipes[1]) * 4};
2781             const PetscInt pipe_hi[3] = {// vertex numbers of pipes
2782                                          (((i + 1) * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j + 1) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + 1 + Npipes[0] + Npipes[1]) * 4};
2783             for (PetscInt dir = 0; dir < 3; dir++) { // x,y,z
2784               const PetscInt ijk[3] = {i, j, k};
2785               for (PetscInt l = 0; l < 4; l++) { // rotations
2786                 cells[J][dir * 2 + 0][l][0] = pipe_lo[dir] + l;
2787                 cells[J][dir * 2 + 0][l][1] = Jvoff + jfaces[dir][0][l];
2788                 cells[J][dir * 2 + 0][l][2] = Jvoff + jfaces[dir][0][(l - 1 + 4) % 4];
2789                 cells[J][dir * 2 + 0][l][3] = pipe_lo[dir] + (l - 1 + 4) % 4;
2790                 cells[J][dir * 2 + 1][l][0] = Jvoff + jfaces[dir][1][l];
2791                 cells[J][dir * 2 + 1][l][1] = pipe_hi[dir] + l;
2792                 cells[J][dir * 2 + 1][l][2] = pipe_hi[dir] + (l - 1 + 4) % 4;
2793                 cells[J][dir * 2 + 1][l][3] = Jvoff + jfaces[dir][1][(l - 1 + 4) % 4];
2794                 if (ijk[dir] == 0) {
2795                   edges[numEdges][0] = pipe_lo[dir] + l;
2796                   edges[numEdges][1] = pipe_lo[dir] + (l + 1) % 4;
2797                   edgeSets[numEdges] = dir * 2 + 1;
2798                   numEdges++;
2799                 }
2800                 if (ijk[dir] + 1 == extent[dir]) {
2801                   edges[numEdges][0] = pipe_hi[dir] + l;
2802                   edges[numEdges][1] = pipe_hi[dir] + (l + 1) % 4;
2803                   edgeSets[numEdges] = dir * 2 + 2;
2804                   numEdges++;
2805                 }
2806               }
2807             }
2808           }
2809         }
2810       }
2811       PetscCheck(numEdges == Ncuts * 4, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Edge count %" PetscInt_FMT " incompatible with number of cuts %" PetscInt_FMT, numEdges, Ncuts);
2812       numFaces   = 24 * Njunctions;
2813       cells_flat = cells[0][0][0];
2814     }
2815     evalFunc   = TPSEvaluate_SchwarzP;
2816     normalFunc = TPSExtrudeNormalFunc_SchwarzP;
2817     break;
2818   case DMPLEX_TPS_GYROID:
2819     if (rank == 0) {
2820       // This is a coarse mesh approximation of the gyroid shifted to being the zero of the level set
2821       //
2822       //     sin(pi*x)*cos(pi*(y+1/2)) + sin(pi*(y+1/2))*cos(pi*(z+1/4)) + sin(pi*(z+1/4))*cos(x)
2823       //
2824       // on the cell [0,2]^3.
2825       //
2826       // Think about dividing that cell into four columns, and focus on the column [0,1]x[0,1]x[0,2].
2827       // If you looked at the gyroid in that column at different slices of z you would see that it kind of spins
2828       // like a boomerang:
2829       //
2830       //     z = 0          z = 1/4        z = 1/2        z = 3/4     //
2831       //     -----          -------        -------        -------     //
2832       //                                                              //
2833       //     +       +      +       +      +       +      +   \   +   //
2834       //      \                                   /            \      //
2835       //       \            `-_   _-'            /              }     //
2836       //        *-_            `-'            _-'              /      //
2837       //     +     `-+      +       +      +-'     +      +   /   +   //
2838       //                                                              //
2839       //                                                              //
2840       //     z = 1          z = 5/4        z = 3/2        z = 7/4     //
2841       //     -----          -------        -------        -------     //
2842       //                                                              //
2843       //     +-_     +      +       +      +     _-+      +   /   +   //
2844       //        `-_            _-_            _-`            /        //
2845       //           \        _-'   `-_        /              {         //
2846       //            \                       /                \        //
2847       //     +       +      +       +      +       +      +   \   +   //
2848       //
2849       //
2850       // This course mesh approximates each of these slices by two line segments,
2851       // and then connects the segments in consecutive layers with quadrilateral faces.
2852       // All of the end points of the segments are multiples of 1/4 except for the
2853       // point * in the picture for z = 0 above and the similar points in other layers.
2854       // That point is at (gamma, gamma, 0), where gamma is calculated below.
2855       //
2856       // The column  [1,2]x[1,2]x[0,2] looks the same as this column;
2857       // The columns [1,2]x[0,1]x[0,2] and [0,1]x[1,2]x[0,2] are mirror images.
2858       //
2859       // As for how this method turned into the names given to the vertices:
2860       // that was not systematic, it was just the way it worked out in my handwritten notes.
2861 
2862       PetscInt facesPerBlock = 64;
2863       PetscInt vertsPerBlock = 56;
2864       PetscInt extentPlus[3];
2865       PetscInt numBlocks, numBlocksPlus;
2866       const PetscInt A = 0, B = 1, C = 2, D = 3, E = 4, F = 5, G = 6, H = 7, II = 8, J = 9, K = 10, L = 11, M = 12, N = 13, O = 14, P = 15, Q = 16, R = 17, S = 18, T = 19, U = 20, V = 21, W = 22, X = 23, Y = 24, Z = 25, Ap = 26, Bp = 27, Cp = 28, Dp = 29, Ep = 30, Fp = 31, Gp = 32, Hp = 33, Ip = 34, Jp = 35, Kp = 36, Lp = 37, Mp = 38, Np = 39, Op = 40, Pp = 41, Qp = 42, Rp = 43, Sp = 44, Tp = 45, Up = 46, Vp = 47, Wp = 48, Xp = 49, Yp = 50, Zp = 51, Aq = 52, Bq = 53, Cq = 54, Dq = 55;
2867       const PetscInt pattern[64][4] = {
2868   /* face to vertex within the coarse discretization of a single gyroid block */
2869   /* layer 0 */
2870         {A,           C,           K,           G          },
2871         {C,           B,           II,          K          },
2872         {D,           A,           H,           L          },
2873         {B + 56 * 1,  D,           L,           J          },
2874         {E,           B + 56 * 1,  J,           N          },
2875         {A + 56 * 2,  E,           N,           H + 56 * 2 },
2876         {F,           A + 56 * 2,  G + 56 * 2,  M          },
2877         {B,           F,           M,           II         },
2878  /* layer 1 */
2879         {G,           K,           Q,           O          },
2880         {K,           II,          P,           Q          },
2881         {L,           H,           O + 56 * 1,  R          },
2882         {J,           L,           R,           P          },
2883         {N,           J,           P,           S          },
2884         {H + 56 * 2,  N,           S,           O + 56 * 3 },
2885         {M,           G + 56 * 2,  O + 56 * 2,  T          },
2886         {II,          M,           T,           P          },
2887  /* layer 2 */
2888         {O,           Q,           Y,           U          },
2889         {Q,           P,           W,           Y          },
2890         {R,           O + 56 * 1,  U + 56 * 1,  Ap         },
2891         {P,           R,           Ap,          W          },
2892         {S,           P,           X,           Bp         },
2893         {O + 56 * 3,  S,           Bp,          V + 56 * 1 },
2894         {T,           O + 56 * 2,  V,           Z          },
2895         {P,           T,           Z,           X          },
2896  /* layer 3 */
2897         {U,           Y,           Ep,          Dp         },
2898         {Y,           W,           Cp,          Ep         },
2899         {Ap,          U + 56 * 1,  Dp + 56 * 1, Gp         },
2900         {W,           Ap,          Gp,          Cp         },
2901         {Bp,          X,           Cp + 56 * 2, Fp         },
2902         {V + 56 * 1,  Bp,          Fp,          Dp + 56 * 1},
2903         {Z,           V,           Dp,          Hp         },
2904         {X,           Z,           Hp,          Cp + 56 * 2},
2905  /* layer 4 */
2906         {Dp,          Ep,          Mp,          Kp         },
2907         {Ep,          Cp,          Ip,          Mp         },
2908         {Gp,          Dp + 56 * 1, Lp,          Np         },
2909         {Cp,          Gp,          Np,          Jp         },
2910         {Fp,          Cp + 56 * 2, Jp + 56 * 2, Pp         },
2911         {Dp + 56 * 1, Fp,          Pp,          Lp         },
2912         {Hp,          Dp,          Kp,          Op         },
2913         {Cp + 56 * 2, Hp,          Op,          Ip + 56 * 2},
2914  /* layer 5 */
2915         {Kp,          Mp,          Sp,          Rp         },
2916         {Mp,          Ip,          Qp,          Sp         },
2917         {Np,          Lp,          Rp,          Tp         },
2918         {Jp,          Np,          Tp,          Qp + 56 * 1},
2919         {Pp,          Jp + 56 * 2, Qp + 56 * 3, Up         },
2920         {Lp,          Pp,          Up,          Rp         },
2921         {Op,          Kp,          Rp,          Vp         },
2922         {Ip + 56 * 2, Op,          Vp,          Qp + 56 * 2},
2923  /* layer 6 */
2924         {Rp,          Sp,          Aq,          Yp         },
2925         {Sp,          Qp,          Wp,          Aq         },
2926         {Tp,          Rp,          Yp,          Cq         },
2927         {Qp + 56 * 1, Tp,          Cq,          Wp + 56 * 1},
2928         {Up,          Qp + 56 * 3, Xp + 56 * 1, Dq         },
2929         {Rp,          Up,          Dq,          Zp         },
2930         {Vp,          Rp,          Zp,          Bq         },
2931         {Qp + 56 * 2, Vp,          Bq,          Xp         },
2932  /* layer 7 (the top is the periodic image of the bottom of layer 0) */
2933         {Yp,          Aq,          C + 56 * 4,  A + 56 * 4 },
2934         {Aq,          Wp,          B + 56 * 4,  C + 56 * 4 },
2935         {Cq,          Yp,          A + 56 * 4,  D + 56 * 4 },
2936         {Wp + 56 * 1, Cq,          D + 56 * 4,  B + 56 * 5 },
2937         {Dq,          Xp + 56 * 1, B + 56 * 5,  E + 56 * 4 },
2938         {Zp,          Dq,          E + 56 * 4,  A + 56 * 6 },
2939         {Bq,          Zp,          A + 56 * 6,  F + 56 * 4 },
2940         {Xp,          Bq,          F + 56 * 4,  B + 56 * 4 }
2941       };
2942       const PetscReal gamma                = PetscAcosReal((PetscSqrtReal(3.) - 1.) / PetscSqrtReal(2.)) / PETSC_PI;
2943       const PetscReal patternCoords[56][3] = {
2944         {1.,        0.,        0.  }, /* A  */
2945         {0.,        1.,        0.  }, /* B  */
2946         {gamma,     gamma,     0.  }, /* C  */
2947         {1 + gamma, 1 - gamma, 0.  }, /* D  */
2948         {2 - gamma, 2 - gamma, 0.  }, /* E  */
2949         {1 - gamma, 1 + gamma, 0.  }, /* F  */
2950 
2951         {.5,        0,         .25 }, /* G  */
2952         {1.5,       0.,        .25 }, /* H  */
2953         {.5,        1.,        .25 }, /* II */
2954         {1.5,       1.,        .25 }, /* J  */
2955         {.25,       .5,        .25 }, /* K  */
2956         {1.25,      .5,        .25 }, /* L  */
2957         {.75,       1.5,       .25 }, /* M  */
2958         {1.75,      1.5,       .25 }, /* N  */
2959 
2960         {0.,        0.,        .5  }, /* O  */
2961         {1.,        1.,        .5  }, /* P  */
2962         {gamma,     1 - gamma, .5  }, /* Q  */
2963         {1 + gamma, gamma,     .5  }, /* R  */
2964         {2 - gamma, 1 + gamma, .5  }, /* S  */
2965         {1 - gamma, 2 - gamma, .5  }, /* T  */
2966 
2967         {0.,        .5,        .75 }, /* U  */
2968         {0.,        1.5,       .75 }, /* V  */
2969         {1.,        .5,        .75 }, /* W  */
2970         {1.,        1.5,       .75 }, /* X  */
2971         {.5,        .75,       .75 }, /* Y  */
2972         {.5,        1.75,      .75 }, /* Z  */
2973         {1.5,       .25,       .75 }, /* Ap */
2974         {1.5,       1.25,      .75 }, /* Bp */
2975 
2976         {1.,        0.,        1.  }, /* Cp */
2977         {0.,        1.,        1.  }, /* Dp */
2978         {1 - gamma, 1 - gamma, 1.  }, /* Ep */
2979         {1 + gamma, 1 + gamma, 1.  }, /* Fp */
2980         {2 - gamma, gamma,     1.  }, /* Gp */
2981         {gamma,     2 - gamma, 1.  }, /* Hp */
2982 
2983         {.5,        0.,        1.25}, /* Ip */
2984         {1.5,       0.,        1.25}, /* Jp */
2985         {.5,        1.,        1.25}, /* Kp */
2986         {1.5,       1.,        1.25}, /* Lp */
2987         {.75,       .5,        1.25}, /* Mp */
2988         {1.75,      .5,        1.25}, /* Np */
2989         {.25,       1.5,       1.25}, /* Op */
2990         {1.25,      1.5,       1.25}, /* Pp */
2991 
2992         {0.,        0.,        1.5 }, /* Qp */
2993         {1.,        1.,        1.5 }, /* Rp */
2994         {1 - gamma, gamma,     1.5 }, /* Sp */
2995         {2 - gamma, 1 - gamma, 1.5 }, /* Tp */
2996         {1 + gamma, 2 - gamma, 1.5 }, /* Up */
2997         {gamma,     1 + gamma, 1.5 }, /* Vp */
2998 
2999         {0.,        .5,        1.75}, /* Wp */
3000         {0.,        1.5,       1.75}, /* Xp */
3001         {1.,        .5,        1.75}, /* Yp */
3002         {1.,        1.5,       1.75}, /* Zp */
3003         {.5,        .25,       1.75}, /* Aq */
3004         {.5,        1.25,      1.75}, /* Bq */
3005         {1.5,       .75,       1.75}, /* Cq */
3006         {1.5,       1.75,      1.75}, /* Dq */
3007       };
3008       PetscInt(*cells)[64][4] = NULL;
3009       PetscBool *seen;
3010       PetscInt  *vertToTrueVert;
3011       PetscInt   count;
3012 
3013       for (PetscInt i = 0; i < 3; i++) extentPlus[i] = extent[i] + 1;
3014       numBlocks = 1;
3015       for (PetscInt i = 0; i < 3; i++) numBlocks *= extent[i];
3016       numBlocksPlus = 1;
3017       for (PetscInt i = 0; i < 3; i++) numBlocksPlus *= extentPlus[i];
3018       numFaces = numBlocks * facesPerBlock;
3019       PetscCall(PetscMalloc1(numBlocks, &cells));
3020       PetscCall(PetscCalloc1(numBlocksPlus * vertsPerBlock, &seen));
3021       for (PetscInt k = 0; k < extent[2]; k++) {
3022         for (PetscInt j = 0; j < extent[1]; j++) {
3023           for (PetscInt i = 0; i < extent[0]; i++) {
3024             for (PetscInt f = 0; f < facesPerBlock; f++) {
3025               for (PetscInt v = 0; v < 4; v++) {
3026                 PetscInt vertRaw     = pattern[f][v];
3027                 PetscInt blockidx    = vertRaw / 56;
3028                 PetscInt patternvert = vertRaw % 56;
3029                 PetscInt xplus       = (blockidx & 1);
3030                 PetscInt yplus       = (blockidx & 2) >> 1;
3031                 PetscInt zplus       = (blockidx & 4) >> 2;
3032                 PetscInt zcoord      = (periodic && periodic[2] == DM_BOUNDARY_PERIODIC) ? ((k + zplus) % extent[2]) : (k + zplus);
3033                 PetscInt ycoord      = (periodic && periodic[1] == DM_BOUNDARY_PERIODIC) ? ((j + yplus) % extent[1]) : (j + yplus);
3034                 PetscInt xcoord      = (periodic && periodic[0] == DM_BOUNDARY_PERIODIC) ? ((i + xplus) % extent[0]) : (i + xplus);
3035                 PetscInt vert        = ((zcoord * extentPlus[1] + ycoord) * extentPlus[0] + xcoord) * 56 + patternvert;
3036 
3037                 cells[(k * extent[1] + j) * extent[0] + i][f][v] = vert;
3038                 seen[vert]                                       = PETSC_TRUE;
3039               }
3040             }
3041           }
3042         }
3043       }
3044       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++)
3045         if (seen[i]) numVertices++;
3046       count = 0;
3047       PetscCall(PetscMalloc1(numBlocksPlus * vertsPerBlock, &vertToTrueVert));
3048       PetscCall(PetscMalloc1(numVertices * 3, &vtxCoords));
3049       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) vertToTrueVert[i] = -1;
3050       for (PetscInt k = 0; k < extentPlus[2]; k++) {
3051         for (PetscInt j = 0; j < extentPlus[1]; j++) {
3052           for (PetscInt i = 0; i < extentPlus[0]; i++) {
3053             for (PetscInt v = 0; v < vertsPerBlock; v++) {
3054               PetscInt vIdx = ((k * extentPlus[1] + j) * extentPlus[0] + i) * vertsPerBlock + v;
3055 
3056               if (seen[vIdx]) {
3057                 PetscInt thisVert;
3058 
3059                 vertToTrueVert[vIdx] = thisVert = count++;
3060 
3061                 for (PetscInt d = 0; d < 3; d++) vtxCoords[3 * thisVert + d] = patternCoords[v][d];
3062                 vtxCoords[3 * thisVert + 0] += i * 2;
3063                 vtxCoords[3 * thisVert + 1] += j * 2;
3064                 vtxCoords[3 * thisVert + 2] += k * 2;
3065               }
3066             }
3067           }
3068         }
3069       }
3070       for (PetscInt i = 0; i < numBlocks; i++) {
3071         for (PetscInt f = 0; f < facesPerBlock; f++) {
3072           for (PetscInt v = 0; v < 4; v++) cells[i][f][v] = vertToTrueVert[cells[i][f][v]];
3073         }
3074       }
3075       PetscCall(PetscFree(vertToTrueVert));
3076       PetscCall(PetscFree(seen));
3077       cells_flat = cells[0][0];
3078       numEdges   = 0;
3079       for (PetscInt i = 0; i < numFaces; i++) {
3080         for (PetscInt e = 0; e < 4; e++) {
3081           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3082           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3083 
3084           for (PetscInt d = 0; d < 3; d++) {
3085             if (!periodic || periodic[0] != DM_BOUNDARY_PERIODIC) {
3086               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) numEdges++;
3087               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) numEdges++;
3088             }
3089           }
3090         }
3091       }
3092       PetscCall(PetscMalloc1(numEdges, &edges));
3093       PetscCall(PetscMalloc1(numEdges, &edgeSets));
3094       for (PetscInt edge = 0, i = 0; i < numFaces; i++) {
3095         for (PetscInt e = 0; e < 4; e++) {
3096           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3097           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3098 
3099           for (PetscInt d = 0; d < 3; d++) {
3100             if (!periodic || periodic[d] != DM_BOUNDARY_PERIODIC) {
3101               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) {
3102                 edges[edge][0]   = ev[0];
3103                 edges[edge][1]   = ev[1];
3104                 edgeSets[edge++] = 2 * d;
3105               }
3106               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) {
3107                 edges[edge][0]   = ev[0];
3108                 edges[edge][1]   = ev[1];
3109                 edgeSets[edge++] = 2 * d + 1;
3110               }
3111             }
3112           }
3113         }
3114       }
3115     }
3116     evalFunc   = TPSEvaluate_Gyroid;
3117     normalFunc = TPSExtrudeNormalFunc_Gyroid;
3118     break;
3119   }
3120 
3121   PetscCall(DMSetDimension(dm, topoDim));
3122   if (rank == 0) PetscCall(DMPlexBuildFromCellList(dm, numFaces, numVertices, 4, cells_flat));
3123   else PetscCall(DMPlexBuildFromCellList(dm, 0, 0, 0, NULL));
3124   PetscCall(PetscFree(cells_flat));
3125   {
3126     DM idm;
3127     PetscCall(DMPlexInterpolate(dm, &idm));
3128     PetscCall(DMPlexReplace_Internal(dm, &idm));
3129   }
3130   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, vtxCoords));
3131   else PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, NULL));
3132   PetscCall(PetscFree(vtxCoords));
3133 
3134   PetscCall(DMCreateLabel(dm, "Face Sets"));
3135   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3136   for (PetscInt e = 0; e < numEdges; e++) {
3137     PetscInt        njoin;
3138     const PetscInt *join, verts[] = {numFaces + edges[e][0], numFaces + edges[e][1]};
3139     PetscCall(DMPlexGetJoin(dm, 2, verts, &njoin, &join));
3140     PetscCheck(njoin == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected unique join of vertices %" PetscInt_FMT " and %" PetscInt_FMT, edges[e][0], edges[e][1]);
3141     PetscCall(DMLabelSetValue(label, join[0], edgeSets[e]));
3142     PetscCall(DMPlexRestoreJoin(dm, 2, verts, &njoin, &join));
3143   }
3144   PetscCall(PetscFree(edges));
3145   PetscCall(PetscFree(edgeSets));
3146   if (tps_distribute) {
3147     DM               pdm = NULL;
3148     PetscPartitioner part;
3149 
3150     PetscCall(DMPlexGetPartitioner(dm, &part));
3151     PetscCall(PetscPartitionerSetFromOptions(part));
3152     PetscCall(DMPlexDistribute(dm, 0, NULL, &pdm));
3153     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
3154     // Do not auto-distribute again
3155     PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
3156   }
3157 
3158   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3159   for (PetscInt refine = 0; refine < refinements; refine++) {
3160     PetscInt     m;
3161     DM           dmf;
3162     Vec          X;
3163     PetscScalar *x;
3164     PetscCall(DMRefine(dm, MPI_COMM_NULL, &dmf));
3165     PetscCall(DMPlexReplace_Internal(dm, &dmf));
3166 
3167     PetscCall(DMGetCoordinatesLocal(dm, &X));
3168     PetscCall(VecGetLocalSize(X, &m));
3169     PetscCall(VecGetArray(X, &x));
3170     for (PetscInt i = 0; i < m; i += 3) PetscCall(TPSNearestPoint(evalFunc, &x[i]));
3171     PetscCall(VecRestoreArray(X, &x));
3172   }
3173 
3174   // Face Sets has already been propagated to new vertices during refinement; this propagates to the initial vertices.
3175   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3176   PetscCall(DMPlexLabelComplete(dm, label));
3177 
3178   if (thickness > 0) {
3179     DM              edm, cdm, ecdm;
3180     DMPlexTransform tr;
3181     const char     *prefix;
3182     PetscOptions    options;
3183     // Code from DMPlexExtrude
3184     PetscCall(DMPlexTransformCreate(PetscObjectComm((PetscObject)dm), &tr));
3185     PetscCall(DMPlexTransformSetDM(tr, dm));
3186     PetscCall(DMPlexTransformSetType(tr, DMPLEXEXTRUDE));
3187     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
3188     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)tr, prefix));
3189     PetscCall(PetscObjectGetOptions((PetscObject)dm, &options));
3190     PetscCall(PetscObjectSetOptions((PetscObject)tr, options));
3191     PetscCall(DMPlexTransformExtrudeSetLayers(tr, layers));
3192     PetscCall(DMPlexTransformExtrudeSetThickness(tr, thickness));
3193     PetscCall(DMPlexTransformExtrudeSetTensor(tr, PETSC_FALSE));
3194     PetscCall(DMPlexTransformExtrudeSetSymmetric(tr, PETSC_TRUE));
3195     PetscCall(DMPlexTransformExtrudeSetNormalFunction(tr, normalFunc));
3196     PetscCall(DMPlexTransformSetFromOptions(tr));
3197     PetscCall(PetscObjectSetOptions((PetscObject)tr, NULL));
3198     PetscCall(DMPlexTransformSetUp(tr));
3199     PetscCall(PetscObjectViewFromOptions((PetscObject)tr, NULL, "-dm_plex_tps_transform_view"));
3200     PetscCall(DMPlexTransformApply(tr, dm, &edm));
3201     PetscCall(DMCopyDisc(dm, edm));
3202     PetscCall(DMGetCoordinateDM(dm, &cdm));
3203     PetscCall(DMGetCoordinateDM(edm, &ecdm));
3204     PetscCall(DMCopyDisc(cdm, ecdm));
3205     PetscCall(DMPlexTransformCreateDiscLabels(tr, edm));
3206     PetscCall(DMPlexTransformDestroy(&tr));
3207     if (edm) {
3208       ((DM_Plex *)edm->data)->printFEM    = ((DM_Plex *)dm->data)->printFEM;
3209       ((DM_Plex *)edm->data)->printL2     = ((DM_Plex *)dm->data)->printL2;
3210       ((DM_Plex *)edm->data)->printLocate = ((DM_Plex *)dm->data)->printLocate;
3211     }
3212     PetscCall(DMPlexReplace_Internal(dm, &edm));
3213   }
3214   PetscFunctionReturn(0);
3215 }
3216 
3217 /*@
3218   DMPlexCreateTPSMesh - Create a distributed, interpolated mesh of a triply-periodic surface
3219 
3220   Collective
3221 
3222   Input Parameters:
3223 + comm   - The communicator for the DM object
3224 . tpstype - Type of triply-periodic surface
3225 . extent - Array of length 3 containing number of periods in each direction
3226 . periodic - array of length 3 with periodicity, or NULL for non-periodic
3227 . tps_distribute - Distribute 2D manifold mesh prior to refinement and extrusion (more scalable)
3228 . refinements - Number of factor-of-2 refinements of 2D manifold mesh
3229 . layers - Number of cell layers extruded in normal direction
3230 - thickness - Thickness in normal direction
3231 
3232   Output Parameter:
3233 . dm  - The DM object
3234 
3235   Notes:
3236   This meshes the surface of the Schwarz P or Gyroid surfaces.  Schwarz P is is the simplest member of the triply-periodic minimal surfaces.
3237   https://en.wikipedia.org/wiki/Schwarz_minimal_surface#Schwarz_P_(%22Primitive%22) and can be cut with "clean" boundaries.
3238   The Gyroid (https://en.wikipedia.org/wiki/Gyroid) is another triply-periodic minimal surface with applications in additive manufacturing; it is much more difficult to "cut" since there are no planes of symmetry.
3239   Our implementation creates a very coarse mesh of the surface and refines (by 4-way splitting) as many times as requested.
3240   On each refinement, all vertices are projected to their nearest point on the surface.
3241   This projection could readily be extended to related surfaces.
3242 
3243   The face (edge) sets for the Schwarz P surface are numbered 1(-x), 2(+x), 3(-y), 4(+y), 5(-z), 6(+z).
3244   When the mesh is refined, "Face Sets" contain the new vertices (created during refinement).  Use DMPlexLabelComplete() to propagate to coarse-level vertices.
3245 
3246   References:
3247 . * - Maskery et al, Insights into the mechanical properties of several triply periodic minimal surface lattice structures made by polymer additive manufacturing, 2017. https://doi.org/10.1016/j.polymer.2017.11.049
3248 
3249   Developer Notes:
3250   The Gyroid mesh does not currently mark boundary sets.
3251 
3252   Level: beginner
3253 
3254 .seealso: `DMPlexCreateSphereMesh()`, `DMSetType()`, `DMCreate()`
3255 @*/
3256 PetscErrorCode DMPlexCreateTPSMesh(MPI_Comm comm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness, DM *dm)
3257 {
3258   PetscFunctionBegin;
3259   PetscCall(DMCreate(comm, dm));
3260   PetscCall(DMSetType(*dm, DMPLEX));
3261   PetscCall(DMPlexCreateTPSMesh_Internal(*dm, tpstype, extent, periodic, tps_distribute, refinements, layers, thickness));
3262   PetscFunctionReturn(0);
3263 }
3264 
3265 /*@
3266   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
3267 
3268   Collective
3269 
3270   Input Parameters:
3271 + comm    - The communicator for the DM object
3272 . dim     - The dimension
3273 . simplex - Use simplices, or tensor product cells
3274 - R       - The radius
3275 
3276   Output Parameter:
3277 . dm  - The DM object
3278 
3279   Level: beginner
3280 
3281 .seealso: `DMPlexCreateBallMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3282 @*/
3283 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscReal R, DM *dm)
3284 {
3285   PetscFunctionBegin;
3286   PetscValidPointer(dm, 5);
3287   PetscCall(DMCreate(comm, dm));
3288   PetscCall(DMSetType(*dm, DMPLEX));
3289   PetscCall(DMPlexCreateSphereMesh_Internal(*dm, dim, simplex, R));
3290   PetscFunctionReturn(0);
3291 }
3292 
3293 static PetscErrorCode DMPlexCreateBallMesh_Internal(DM dm, PetscInt dim, PetscReal R)
3294 {
3295   DM      sdm, vol;
3296   DMLabel bdlabel;
3297 
3298   PetscFunctionBegin;
3299   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
3300   PetscCall(DMSetType(sdm, DMPLEX));
3301   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sdm, "bd_"));
3302   PetscCall(DMPlexCreateSphereMesh_Internal(sdm, dim - 1, PETSC_TRUE, R));
3303   PetscCall(DMSetFromOptions(sdm));
3304   PetscCall(DMViewFromOptions(sdm, NULL, "-dm_view"));
3305   PetscCall(DMPlexGenerate(sdm, NULL, PETSC_TRUE, &vol));
3306   PetscCall(DMDestroy(&sdm));
3307   PetscCall(DMPlexReplace_Internal(dm, &vol));
3308   PetscCall(DMCreateLabel(dm, "marker"));
3309   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
3310   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
3311   PetscCall(DMPlexLabelComplete(dm, bdlabel));
3312   PetscFunctionReturn(0);
3313 }
3314 
3315 /*@
3316   DMPlexCreateBallMesh - Creates a simplex mesh on the d-dimensional ball, B^d.
3317 
3318   Collective
3319 
3320   Input Parameters:
3321 + comm  - The communicator for the DM object
3322 . dim   - The dimension
3323 - R     - The radius
3324 
3325   Output Parameter:
3326 . dm  - The DM object
3327 
3328   Options Database Keys:
3329 - bd_dm_refine - This will refine the surface mesh preserving the sphere geometry
3330 
3331   Level: beginner
3332 
3333 .seealso: `DMPlexCreateSphereMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3334 @*/
3335 PetscErrorCode DMPlexCreateBallMesh(MPI_Comm comm, PetscInt dim, PetscReal R, DM *dm)
3336 {
3337   PetscFunctionBegin;
3338   PetscCall(DMCreate(comm, dm));
3339   PetscCall(DMSetType(*dm, DMPLEX));
3340   PetscCall(DMPlexCreateBallMesh_Internal(*dm, dim, R));
3341   PetscFunctionReturn(0);
3342 }
3343 
3344 static PetscErrorCode DMPlexCreateReferenceCell_Internal(DM rdm, DMPolytopeType ct)
3345 {
3346   PetscFunctionBegin;
3347   switch (ct) {
3348   case DM_POLYTOPE_POINT: {
3349     PetscInt    numPoints[1]        = {1};
3350     PetscInt    coneSize[1]         = {0};
3351     PetscInt    cones[1]            = {0};
3352     PetscInt    coneOrientations[1] = {0};
3353     PetscScalar vertexCoords[1]     = {0.0};
3354 
3355     PetscCall(DMSetDimension(rdm, 0));
3356     PetscCall(DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3357   } break;
3358   case DM_POLYTOPE_SEGMENT: {
3359     PetscInt    numPoints[2]        = {2, 1};
3360     PetscInt    coneSize[3]         = {2, 0, 0};
3361     PetscInt    cones[2]            = {1, 2};
3362     PetscInt    coneOrientations[2] = {0, 0};
3363     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3364 
3365     PetscCall(DMSetDimension(rdm, 1));
3366     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3367   } break;
3368   case DM_POLYTOPE_POINT_PRISM_TENSOR: {
3369     PetscInt    numPoints[2]        = {2, 1};
3370     PetscInt    coneSize[3]         = {2, 0, 0};
3371     PetscInt    cones[2]            = {1, 2};
3372     PetscInt    coneOrientations[2] = {0, 0};
3373     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3374 
3375     PetscCall(DMSetDimension(rdm, 1));
3376     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3377   } break;
3378   case DM_POLYTOPE_TRIANGLE: {
3379     PetscInt    numPoints[2]        = {3, 1};
3380     PetscInt    coneSize[4]         = {3, 0, 0, 0};
3381     PetscInt    cones[3]            = {1, 2, 3};
3382     PetscInt    coneOrientations[3] = {0, 0, 0};
3383     PetscScalar vertexCoords[6]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0};
3384 
3385     PetscCall(DMSetDimension(rdm, 2));
3386     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3387   } break;
3388   case DM_POLYTOPE_QUADRILATERAL: {
3389     PetscInt    numPoints[2]        = {4, 1};
3390     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3391     PetscInt    cones[4]            = {1, 2, 3, 4};
3392     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3393     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0};
3394 
3395     PetscCall(DMSetDimension(rdm, 2));
3396     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3397   } break;
3398   case DM_POLYTOPE_SEG_PRISM_TENSOR: {
3399     PetscInt    numPoints[2]        = {4, 1};
3400     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3401     PetscInt    cones[4]            = {1, 2, 3, 4};
3402     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3403     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0};
3404 
3405     PetscCall(DMSetDimension(rdm, 2));
3406     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3407   } break;
3408   case DM_POLYTOPE_TETRAHEDRON: {
3409     PetscInt    numPoints[2]        = {4, 1};
3410     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3411     PetscInt    cones[4]            = {1, 2, 3, 4};
3412     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3413     PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0};
3414 
3415     PetscCall(DMSetDimension(rdm, 3));
3416     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3417   } break;
3418   case DM_POLYTOPE_HEXAHEDRON: {
3419     PetscInt    numPoints[2]        = {8, 1};
3420     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3421     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3422     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3423     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3424 
3425     PetscCall(DMSetDimension(rdm, 3));
3426     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3427   } break;
3428   case DM_POLYTOPE_TRI_PRISM: {
3429     PetscInt    numPoints[2]        = {6, 1};
3430     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3431     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3432     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3433     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3434 
3435     PetscCall(DMSetDimension(rdm, 3));
3436     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3437   } break;
3438   case DM_POLYTOPE_TRI_PRISM_TENSOR: {
3439     PetscInt    numPoints[2]        = {6, 1};
3440     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3441     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3442     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3443     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3444 
3445     PetscCall(DMSetDimension(rdm, 3));
3446     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3447   } break;
3448   case DM_POLYTOPE_QUAD_PRISM_TENSOR: {
3449     PetscInt    numPoints[2]        = {8, 1};
3450     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3451     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3452     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3453     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3454 
3455     PetscCall(DMSetDimension(rdm, 3));
3456     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3457   } break;
3458   case DM_POLYTOPE_PYRAMID: {
3459     PetscInt    numPoints[2]        = {5, 1};
3460     PetscInt    coneSize[6]         = {5, 0, 0, 0, 0, 0};
3461     PetscInt    cones[5]            = {1, 2, 3, 4, 5};
3462     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3463     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, 0.0, 0.0, 1.0};
3464 
3465     PetscCall(DMSetDimension(rdm, 3));
3466     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3467   } break;
3468   default:
3469     SETERRQ(PetscObjectComm((PetscObject)rdm), PETSC_ERR_ARG_WRONG, "Cannot create reference cell for cell type %s", DMPolytopeTypes[ct]);
3470   }
3471   {
3472     PetscInt Nv, v;
3473 
3474     /* Must create the celltype label here so that we do not automatically try to compute the types */
3475     PetscCall(DMCreateLabel(rdm, "celltype"));
3476     PetscCall(DMPlexSetCellType(rdm, 0, ct));
3477     PetscCall(DMPlexGetChart(rdm, NULL, &Nv));
3478     for (v = 1; v < Nv; ++v) PetscCall(DMPlexSetCellType(rdm, v, DM_POLYTOPE_POINT));
3479   }
3480   PetscCall(DMPlexInterpolateInPlace_Internal(rdm));
3481   PetscCall(PetscObjectSetName((PetscObject)rdm, DMPolytopeTypes[ct]));
3482   PetscFunctionReturn(0);
3483 }
3484 
3485 /*@
3486   DMPlexCreateReferenceCell - Create a DMPLEX with the appropriate FEM reference cell
3487 
3488   Collective
3489 
3490   Input Parameters:
3491 + comm - The communicator
3492 - ct   - The cell type of the reference cell
3493 
3494   Output Parameter:
3495 . refdm - The reference cell
3496 
3497   Level: intermediate
3498 
3499 .seealso: `DMPlexCreateReferenceCell()`, `DMPlexCreateBoxMesh()`
3500 @*/
3501 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, DMPolytopeType ct, DM *refdm)
3502 {
3503   PetscFunctionBegin;
3504   PetscCall(DMCreate(comm, refdm));
3505   PetscCall(DMSetType(*refdm, DMPLEX));
3506   PetscCall(DMPlexCreateReferenceCell_Internal(*refdm, ct));
3507   PetscFunctionReturn(0);
3508 }
3509 
3510 static PetscErrorCode DMPlexCreateBoundaryLabel_Private(DM dm, const char name[])
3511 {
3512   DM        plex;
3513   DMLabel   label;
3514   PetscBool hasLabel;
3515 
3516   PetscFunctionBegin;
3517   PetscCall(DMHasLabel(dm, name, &hasLabel));
3518   if (hasLabel) PetscFunctionReturn(0);
3519   PetscCall(DMCreateLabel(dm, name));
3520   PetscCall(DMGetLabel(dm, name, &label));
3521   PetscCall(DMConvert(dm, DMPLEX, &plex));
3522   PetscCall(DMPlexMarkBoundaryFaces(plex, 1, label));
3523   PetscCall(DMPlexLabelComplete(plex, label));
3524   PetscCall(DMDestroy(&plex));
3525   PetscFunctionReturn(0);
3526 }
3527 
3528 /*
3529   We use the last coordinate as the radius, the inner radius is lower[dim-1] and the outer radius is upper[dim-1]. Then we map the first coordinate around the circle.
3530 
3531     (x, y) -> (r, theta) = (x[1], (x[0] - lower[0]) * 2\pi/(upper[0] - lower[0]))
3532 */
3533 static void boxToAnnulus(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
3534 {
3535   const PetscReal low = PetscRealPart(constants[0]);
3536   const PetscReal upp = PetscRealPart(constants[1]);
3537   const PetscReal r   = PetscRealPart(u[1]);
3538   const PetscReal th  = 2. * PETSC_PI * (PetscRealPart(u[0]) - low) / (upp - low);
3539 
3540   f0[0] = r * PetscCosReal(th);
3541   f0[1] = r * PetscSinReal(th);
3542 }
3543 
3544 const char *const DMPlexShapes[] = {"box", "box_surface", "ball", "sphere", "cylinder", "schwarz_p", "gyroid", "doublet", "annulus", "unknown", "DMPlexShape", "DM_SHAPE_", NULL};
3545 
3546 static PetscErrorCode DMPlexCreateFromOptions_Internal(PetscOptionItems *PetscOptionsObject, PetscBool *useCoordSpace, DM dm)
3547 {
3548   DMPlexShape    shape   = DM_SHAPE_BOX;
3549   DMPolytopeType cell    = DM_POLYTOPE_TRIANGLE;
3550   PetscInt       dim     = 2;
3551   PetscBool      simplex = PETSC_TRUE, interpolate = PETSC_TRUE, adjCone = PETSC_FALSE, adjClosure = PETSC_TRUE, refDomain = PETSC_FALSE;
3552   PetscBool      flg, flg2, fflg, bdfflg, nameflg;
3553   MPI_Comm       comm;
3554   char           filename[PETSC_MAX_PATH_LEN]   = "<unspecified>";
3555   char           bdFilename[PETSC_MAX_PATH_LEN] = "<unspecified>";
3556   char           plexname[PETSC_MAX_PATH_LEN]   = "";
3557 
3558   PetscFunctionBegin;
3559   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3560   /* TODO Turn this into a registration interface */
3561   PetscCall(PetscOptionsString("-dm_plex_filename", "File containing a mesh", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &fflg));
3562   PetscCall(PetscOptionsString("-dm_plex_boundary_filename", "File containing a mesh boundary", "DMPlexCreateFromFile", bdFilename, bdFilename, sizeof(bdFilename), &bdfflg));
3563   PetscCall(PetscOptionsString("-dm_plex_name", "Name of the mesh in the file", "DMPlexCreateFromFile", plexname, plexname, sizeof(plexname), &nameflg));
3564   PetscCall(PetscOptionsEnum("-dm_plex_cell", "Cell shape", "", DMPolytopeTypes, (PetscEnum)cell, (PetscEnum *)&cell, NULL));
3565   PetscCall(PetscOptionsBool("-dm_plex_reference_cell_domain", "Use a reference cell domain", "", refDomain, &refDomain, NULL));
3566   PetscCall(PetscOptionsEnum("-dm_plex_shape", "Shape for built-in mesh", "", DMPlexShapes, (PetscEnum)shape, (PetscEnum *)&shape, &flg));
3567   PetscCall(PetscOptionsBoundedInt("-dm_plex_dim", "Topological dimension of the mesh", "DMGetDimension", dim, &dim, &flg, 0));
3568   PetscCheck(!(dim < 0) && !(dim > 3), comm, PETSC_ERR_ARG_OUTOFRANGE, "Dimension %" PetscInt_FMT " should be in [1, 3]", dim);
3569   PetscCall(PetscOptionsBool("-dm_plex_simplex", "Mesh cell shape", "", simplex, &simplex, &flg));
3570   PetscCall(PetscOptionsBool("-dm_plex_interpolate", "Flag to create edges and faces automatically", "", interpolate, &interpolate, &flg));
3571   PetscCall(PetscOptionsBool("-dm_plex_adj_cone", "Set adjacency direction", "DMSetBasicAdjacency", adjCone, &adjCone, &flg));
3572   PetscCall(PetscOptionsBool("-dm_plex_adj_closure", "Set adjacency size", "DMSetBasicAdjacency", adjClosure, &adjClosure, &flg2));
3573   if (flg || flg2) PetscCall(DMSetBasicAdjacency(dm, adjCone, adjClosure));
3574 
3575   switch (cell) {
3576   case DM_POLYTOPE_POINT:
3577   case DM_POLYTOPE_SEGMENT:
3578   case DM_POLYTOPE_POINT_PRISM_TENSOR:
3579   case DM_POLYTOPE_TRIANGLE:
3580   case DM_POLYTOPE_QUADRILATERAL:
3581   case DM_POLYTOPE_TETRAHEDRON:
3582   case DM_POLYTOPE_HEXAHEDRON:
3583     *useCoordSpace = PETSC_TRUE;
3584     break;
3585   default:
3586     *useCoordSpace = PETSC_FALSE;
3587     break;
3588   }
3589 
3590   if (fflg) {
3591     DM dmnew;
3592 
3593     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), filename, plexname, interpolate, &dmnew));
3594     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3595     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3596   } else if (refDomain) {
3597     PetscCall(DMPlexCreateReferenceCell_Internal(dm, cell));
3598   } else if (bdfflg) {
3599     DM bdm, dmnew;
3600 
3601     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), bdFilename, plexname, interpolate, &bdm));
3602     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)bdm, "bd_"));
3603     PetscCall(DMSetFromOptions(bdm));
3604     PetscCall(DMPlexGenerate(bdm, NULL, interpolate, &dmnew));
3605     PetscCall(DMDestroy(&bdm));
3606     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3607     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3608   } else {
3609     PetscCall(PetscObjectSetName((PetscObject)dm, DMPlexShapes[shape]));
3610     switch (shape) {
3611     case DM_SHAPE_BOX:
3612     case DM_SHAPE_ANNULUS: {
3613       PetscInt       faces[3]  = {0, 0, 0};
3614       PetscReal      lower[3]  = {0, 0, 0};
3615       PetscReal      upper[3]  = {1, 1, 1};
3616       DMBoundaryType bdt[3]    = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3617       PetscBool      isAnnular = shape == DM_SHAPE_ANNULUS ? PETSC_TRUE : PETSC_FALSE;
3618       PetscInt       i, n;
3619 
3620       n = dim;
3621       for (i = 0; i < dim; ++i) faces[i] = (dim == 1 ? 1 : 4 - dim);
3622       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3623       n = 3;
3624       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3625       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3626       n = 3;
3627       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3628       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3629       n = 3;
3630       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
3631       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3632 
3633       PetscCheck(!isAnnular || dim == 2, comm, PETSC_ERR_ARG_OUTOFRANGE, "Only two dimensional annuli have been implemented");
3634       if (isAnnular)
3635         for (i = 0; i < dim - 1; ++i) bdt[i] = DM_BOUNDARY_PERIODIC;
3636 
3637       switch (cell) {
3638       case DM_POLYTOPE_TRI_PRISM_TENSOR:
3639         PetscCall(DMPlexCreateWedgeBoxMesh_Internal(dm, faces, lower, upper, bdt));
3640         if (!interpolate) {
3641           DM udm;
3642 
3643           PetscCall(DMPlexUninterpolate(dm, &udm));
3644           PetscCall(DMPlexReplace_Internal(dm, &udm));
3645         }
3646         break;
3647       default:
3648         PetscCall(DMPlexCreateBoxMesh_Internal(dm, dim, simplex, faces, lower, upper, bdt, interpolate));
3649         break;
3650       }
3651       if (isAnnular) {
3652         DM          cdm;
3653         PetscDS     cds;
3654         PetscScalar bounds[2] = {lower[0], upper[0]};
3655 
3656         // Fix coordinates for annular region
3657         PetscCall(DMSetPeriodicity(dm, NULL, NULL, NULL));
3658         PetscCall(DMSetCellCoordinatesLocal(dm, NULL));
3659         PetscCall(DMSetCellCoordinates(dm, NULL));
3660         PetscCall(DMPlexCreateCoordinateSpace(dm, 1, NULL));
3661         PetscCall(DMGetCoordinateDM(dm, &cdm));
3662         PetscCall(DMGetDS(cdm, &cds));
3663         PetscCall(PetscDSSetConstants(cds, 2, bounds));
3664         PetscCall(DMPlexRemapGeometry(dm, 0.0, boxToAnnulus));
3665       }
3666     } break;
3667     case DM_SHAPE_BOX_SURFACE: {
3668       PetscInt  faces[3] = {0, 0, 0};
3669       PetscReal lower[3] = {0, 0, 0};
3670       PetscReal upper[3] = {1, 1, 1};
3671       PetscInt  i, n;
3672 
3673       n = dim + 1;
3674       for (i = 0; i < dim + 1; ++i) faces[i] = (dim + 1 == 1 ? 1 : 4 - (dim + 1));
3675       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3676       n = 3;
3677       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3678       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3679       n = 3;
3680       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3681       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3682       PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(dm, dim + 1, faces, lower, upper, interpolate));
3683     } break;
3684     case DM_SHAPE_SPHERE: {
3685       PetscReal R = 1.0;
3686 
3687       PetscCall(PetscOptionsReal("-dm_plex_sphere_radius", "Radius of the sphere", "", R, &R, &flg));
3688       PetscCall(DMPlexCreateSphereMesh_Internal(dm, dim, simplex, R));
3689     } break;
3690     case DM_SHAPE_BALL: {
3691       PetscReal R = 1.0;
3692 
3693       PetscCall(PetscOptionsReal("-dm_plex_ball_radius", "Radius of the ball", "", R, &R, &flg));
3694       PetscCall(DMPlexCreateBallMesh_Internal(dm, dim, R));
3695     } break;
3696     case DM_SHAPE_CYLINDER: {
3697       DMBoundaryType bdt = DM_BOUNDARY_NONE;
3698       PetscInt       Nw  = 6;
3699 
3700       PetscCall(PetscOptionsEnum("-dm_plex_cylinder_bd", "Boundary type in the z direction", "", DMBoundaryTypes, (PetscEnum)bdt, (PetscEnum *)&bdt, NULL));
3701       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_wedges", "Number of wedges around the cylinder", "", Nw, &Nw, NULL));
3702       switch (cell) {
3703       case DM_POLYTOPE_TRI_PRISM_TENSOR:
3704         PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(dm, Nw, interpolate));
3705         break;
3706       default:
3707         PetscCall(DMPlexCreateHexCylinderMesh_Internal(dm, bdt));
3708         break;
3709       }
3710     } break;
3711     case DM_SHAPE_SCHWARZ_P: // fallthrough
3712     case DM_SHAPE_GYROID: {
3713       PetscInt       extent[3] = {1, 1, 1}, refine = 0, layers = 0, three;
3714       PetscReal      thickness   = 0.;
3715       DMBoundaryType periodic[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3716       DMPlexTPSType  tps_type    = shape == DM_SHAPE_SCHWARZ_P ? DMPLEX_TPS_SCHWARZ_P : DMPLEX_TPS_GYROID;
3717       PetscBool      tps_distribute;
3718       PetscCall(PetscOptionsIntArray("-dm_plex_tps_extent", "Number of replicas for each of three dimensions", NULL, extent, (three = 3, &three), NULL));
3719       PetscCall(PetscOptionsInt("-dm_plex_tps_refine", "Number of refinements", NULL, refine, &refine, NULL));
3720       PetscCall(PetscOptionsEnumArray("-dm_plex_tps_periodic", "Periodicity in each of three dimensions", NULL, DMBoundaryTypes, (PetscEnum *)periodic, (three = 3, &three), NULL));
3721       PetscCall(PetscOptionsInt("-dm_plex_tps_layers", "Number of layers in volumetric extrusion (or zero to not extrude)", NULL, layers, &layers, NULL));
3722       PetscCall(PetscOptionsReal("-dm_plex_tps_thickness", "Thickness of volumetric extrusion", NULL, thickness, &thickness, NULL));
3723       PetscCall(DMPlexDistributeGetDefault(dm, &tps_distribute));
3724       PetscCall(PetscOptionsBool("-dm_plex_tps_distribute", "Distribute the 2D mesh prior to refinement and extrusion", NULL, tps_distribute, &tps_distribute, NULL));
3725       PetscCall(DMPlexCreateTPSMesh_Internal(dm, tps_type, extent, periodic, tps_distribute, refine, layers, thickness));
3726     } break;
3727     case DM_SHAPE_DOUBLET: {
3728       DM        dmnew;
3729       PetscReal rl = 0.0;
3730 
3731       PetscCall(PetscOptionsReal("-dm_plex_doublet_refinementlimit", "Refinement limit", NULL, rl, &rl, NULL));
3732       PetscCall(DMPlexCreateDoublet(PetscObjectComm((PetscObject)dm), dim, simplex, interpolate, rl, &dmnew));
3733       PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3734       PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3735     } break;
3736     default:
3737       SETERRQ(comm, PETSC_ERR_SUP, "Domain shape %s is unsupported", DMPlexShapes[shape]);
3738     }
3739   }
3740   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3741   if (!((PetscObject)dm)->name && nameflg) PetscCall(PetscObjectSetName((PetscObject)dm, plexname));
3742   PetscFunctionReturn(0);
3743 }
3744 
3745 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
3746 {
3747   DM_Plex  *mesh = (DM_Plex *)dm->data;
3748   PetscBool flg, flg2;
3749   char      bdLabel[PETSC_MAX_PATH_LEN];
3750 
3751   PetscFunctionBegin;
3752   /* Handle viewing */
3753   PetscCall(PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL));
3754   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fem", "Debug output level all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL, 0));
3755   PetscCall(PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL));
3756   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL, 0));
3757   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_locate", "Debug output level all point location computations", "DMLocatePoints", 0, &mesh->printLocate, NULL, 0));
3758   PetscCall(DMMonitorSetFromOptions(dm, "-dm_plex_monitor_throughput", "Monitor the simulation throughput", "DMPlexMonitorThroughput", DMPlexMonitorThroughput, NULL, &flg));
3759   if (flg) PetscCall(PetscLogDefaultBegin());
3760   /* Labeling */
3761   PetscCall(PetscOptionsString("-dm_plex_boundary_label", "Label to mark the mesh boundary", "", bdLabel, bdLabel, sizeof(bdLabel), &flg));
3762   if (flg) PetscCall(DMPlexCreateBoundaryLabel_Private(dm, bdLabel));
3763   /* Point Location */
3764   PetscCall(PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL));
3765   /* Partitioning and distribution */
3766   PetscCall(PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL));
3767   /* Generation and remeshing */
3768   PetscCall(PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL));
3769   /* Projection behavior */
3770   PetscCall(PetscOptionsBoundedInt("-dm_plex_max_projection_height", "Maxmimum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL, 0));
3771   PetscCall(PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL));
3772   /* Checking structure */
3773   {
3774     PetscBool all = PETSC_FALSE;
3775 
3776     PetscCall(PetscOptionsBool("-dm_plex_check_all", "Perform all basic checks", "DMPlexCheck", PETSC_FALSE, &all, NULL));
3777     if (all) {
3778       PetscCall(DMPlexCheck(dm));
3779     } else {
3780       PetscCall(PetscOptionsBool("-dm_plex_check_symmetry", "Check that the adjacency information in the mesh is symmetric", "DMPlexCheckSymmetry", PETSC_FALSE, &flg, &flg2));
3781       if (flg && flg2) PetscCall(DMPlexCheckSymmetry(dm));
3782       PetscCall(PetscOptionsBool("-dm_plex_check_skeleton", "Check that each cell has the correct number of vertices (only for homogeneous simplex or tensor meshes)", "DMPlexCheckSkeleton", PETSC_FALSE, &flg, &flg2));
3783       if (flg && flg2) PetscCall(DMPlexCheckSkeleton(dm, 0));
3784       PetscCall(PetscOptionsBool("-dm_plex_check_faces", "Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type", "DMPlexCheckFaces", PETSC_FALSE, &flg, &flg2));
3785       if (flg && flg2) PetscCall(DMPlexCheckFaces(dm, 0));
3786       PetscCall(PetscOptionsBool("-dm_plex_check_geometry", "Check that cells have positive volume", "DMPlexCheckGeometry", PETSC_FALSE, &flg, &flg2));
3787       if (flg && flg2) PetscCall(DMPlexCheckGeometry(dm));
3788       PetscCall(PetscOptionsBool("-dm_plex_check_pointsf", "Check some necessary conditions for PointSF", "DMPlexCheckPointSF", PETSC_FALSE, &flg, &flg2));
3789       if (flg && flg2) PetscCall(DMPlexCheckPointSF(dm, NULL, PETSC_FALSE));
3790       PetscCall(PetscOptionsBool("-dm_plex_check_interface_cones", "Check points on inter-partition interfaces have conforming order of cone points", "DMPlexCheckInterfaceCones", PETSC_FALSE, &flg, &flg2));
3791       if (flg && flg2) PetscCall(DMPlexCheckInterfaceCones(dm));
3792     }
3793     PetscCall(PetscOptionsBool("-dm_plex_check_cell_shape", "Check cell shape", "DMPlexCheckCellShape", PETSC_FALSE, &flg, &flg2));
3794     if (flg && flg2) PetscCall(DMPlexCheckCellShape(dm, PETSC_TRUE, PETSC_DETERMINE));
3795   }
3796   {
3797     PetscReal scale = 1.0;
3798 
3799     PetscCall(PetscOptionsReal("-dm_plex_scale", "Scale factor for mesh coordinates", "DMPlexScale", scale, &scale, &flg));
3800     if (flg) {
3801       Vec coordinates, coordinatesLocal;
3802 
3803       PetscCall(DMGetCoordinates(dm, &coordinates));
3804       PetscCall(DMGetCoordinatesLocal(dm, &coordinatesLocal));
3805       PetscCall(VecScale(coordinates, scale));
3806       PetscCall(VecScale(coordinatesLocal, scale));
3807     }
3808   }
3809   PetscCall(PetscPartitionerSetFromOptions(mesh->partitioner));
3810   PetscFunctionReturn(0);
3811 }
3812 
3813 PetscErrorCode DMSetFromOptions_Overlap_Plex(DM dm, PetscOptionItems *PetscOptionsObject, PetscInt *overlap)
3814 {
3815   PetscInt  numOvLabels = 16, numOvExLabels = 16;
3816   char     *ovLabelNames[16], *ovExLabelNames[16];
3817   PetscInt  numOvValues = 16, numOvExValues = 16, l;
3818   PetscBool flg;
3819 
3820   PetscFunctionBegin;
3821   PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", *overlap, overlap, NULL, 0));
3822   PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_labels", "List of overlap label names", "DMPlexDistribute", ovLabelNames, &numOvLabels, &flg));
3823   if (!flg) numOvLabels = 0;
3824   if (numOvLabels) {
3825     ((DM_Plex *)dm->data)->numOvLabels = numOvLabels;
3826     for (l = 0; l < numOvLabels; ++l) {
3827       PetscCall(DMGetLabel(dm, ovLabelNames[l], &((DM_Plex *)dm->data)->ovLabels[l]));
3828       PetscCheck(((DM_Plex *)dm->data)->ovLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovLabelNames[l]);
3829       PetscCall(PetscFree(ovLabelNames[l]));
3830     }
3831     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_values", "List of overlap label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovValues, &numOvValues, &flg));
3832     if (!flg) numOvValues = 0;
3833     PetscCheck(numOvLabels == numOvValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvLabels, numOvValues);
3834 
3835     PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_exclude_labels", "List of overlap exclude label names", "DMPlexDistribute", ovExLabelNames, &numOvExLabels, &flg));
3836     if (!flg) numOvExLabels = 0;
3837     ((DM_Plex *)dm->data)->numOvExLabels = numOvExLabels;
3838     for (l = 0; l < numOvExLabels; ++l) {
3839       PetscCall(DMGetLabel(dm, ovExLabelNames[l], &((DM_Plex *)dm->data)->ovExLabels[l]));
3840       PetscCheck(((DM_Plex *)dm->data)->ovExLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovExLabelNames[l]);
3841       PetscCall(PetscFree(ovExLabelNames[l]));
3842     }
3843     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_exclude_values", "List of overlap exclude label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovExValues, &numOvExValues, &flg));
3844     if (!flg) numOvExValues = 0;
3845     PetscCheck(numOvExLabels == numOvExValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of exclude labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvExLabels, numOvExValues);
3846   }
3847   PetscFunctionReturn(0);
3848 }
3849 
3850 static PetscErrorCode DMSetFromOptions_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
3851 {
3852   PetscFunctionList        ordlist;
3853   char                     oname[256];
3854   PetscReal                volume    = -1.0;
3855   PetscInt                 prerefine = 0, refine = 0, r, coarsen = 0, overlap = 0, extLayers = 0, dim;
3856   PetscBool                uniformOrig, created = PETSC_FALSE, uniform = PETSC_TRUE, distribute, interpolate = PETSC_TRUE, coordSpace = PETSC_TRUE, remap = PETSC_TRUE, ghostCells = PETSC_FALSE, isHierarchy, ignoreModel = PETSC_FALSE, flg;
3857   DMPlexReorderDefaultFlag reorder;
3858 
3859   PetscFunctionBegin;
3860   PetscOptionsHeadBegin(PetscOptionsObject, "DMPlex Options");
3861   /* Handle automatic creation */
3862   PetscCall(DMGetDimension(dm, &dim));
3863   if (dim < 0) {
3864     PetscCall(DMPlexCreateFromOptions_Internal(PetscOptionsObject, &coordSpace, dm));
3865     created = PETSC_TRUE;
3866   }
3867   PetscCall(DMGetDimension(dm, &dim));
3868   /* Handle interpolation before distribution */
3869   PetscCall(PetscOptionsBool("-dm_plex_interpolate_pre", "Flag to interpolate mesh before distribution", "", interpolate, &interpolate, &flg));
3870   if (flg) {
3871     DMPlexInterpolatedFlag interpolated;
3872 
3873     PetscCall(DMPlexIsInterpolated(dm, &interpolated));
3874     if (interpolated == DMPLEX_INTERPOLATED_FULL && !interpolate) {
3875       DM udm;
3876 
3877       PetscCall(DMPlexUninterpolate(dm, &udm));
3878       PetscCall(DMPlexReplace_Internal(dm, &udm));
3879     } else if (interpolated != DMPLEX_INTERPOLATED_FULL && interpolate) {
3880       DM idm;
3881 
3882       PetscCall(DMPlexInterpolate(dm, &idm));
3883       PetscCall(DMPlexReplace_Internal(dm, &idm));
3884     }
3885   }
3886   /* Handle DMPlex refinement before distribution */
3887   PetscCall(PetscOptionsBool("-dm_refine_ignore_model", "Flag to ignore the geometry model when refining", "DMCreate", ignoreModel, &ignoreModel, &flg));
3888   if (flg) ((DM_Plex *)dm->data)->ignoreModel = ignoreModel;
3889   PetscCall(DMPlexGetRefinementUniform(dm, &uniformOrig));
3890   PetscCall(PetscOptionsBoundedInt("-dm_refine_pre", "The number of refinements before distribution", "DMCreate", prerefine, &prerefine, NULL, 0));
3891   PetscCall(PetscOptionsBool("-dm_refine_remap_pre", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
3892   PetscCall(PetscOptionsBool("-dm_refine_uniform_pre", "Flag for uniform refinement before distribution", "DMCreate", uniform, &uniform, &flg));
3893   if (flg) PetscCall(DMPlexSetRefinementUniform(dm, uniform));
3894   PetscCall(PetscOptionsReal("-dm_refine_volume_limit_pre", "The maximum cell volume after refinement before distribution", "DMCreate", volume, &volume, &flg));
3895   if (flg) {
3896     PetscCall(DMPlexSetRefinementUniform(dm, PETSC_FALSE));
3897     PetscCall(DMPlexSetRefinementLimit(dm, volume));
3898     prerefine = PetscMax(prerefine, 1);
3899   }
3900   for (r = 0; r < prerefine; ++r) {
3901     DM             rdm;
3902     PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
3903 
3904     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3905     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
3906     PetscCall(DMPlexReplace_Internal(dm, &rdm));
3907     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3908     if (coordFunc && remap) {
3909       PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
3910       ((DM_Plex *)dm->data)->coordFunc = coordFunc;
3911     }
3912   }
3913   PetscCall(DMPlexSetRefinementUniform(dm, uniformOrig));
3914   /* Handle DMPlex extrusion before distribution */
3915   PetscCall(PetscOptionsBoundedInt("-dm_extrude", "The number of layers to extrude", "", extLayers, &extLayers, NULL, 0));
3916   if (extLayers) {
3917     DM edm;
3918 
3919     PetscCall(DMExtrude(dm, extLayers, &edm));
3920     PetscCall(DMPlexReplace_Internal(dm, &edm));
3921     ((DM_Plex *)dm->data)->coordFunc = NULL;
3922     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3923     extLayers = 0;
3924   }
3925   /* Handle DMPlex reordering before distribution */
3926   PetscCall(DMPlexReorderGetDefault(dm, &reorder));
3927   PetscCall(MatGetOrderingList(&ordlist));
3928   PetscCall(PetscStrncpy(oname, MATORDERINGNATURAL, sizeof(oname)));
3929   PetscCall(PetscOptionsFList("-dm_plex_reorder", "Set mesh reordering type", "DMPlexGetOrdering", ordlist, MATORDERINGNATURAL, oname, sizeof(oname), &flg));
3930   if (reorder == DMPLEX_REORDER_DEFAULT_TRUE || flg) {
3931     DM pdm;
3932     IS perm;
3933 
3934     PetscCall(DMPlexGetOrdering(dm, oname, NULL, &perm));
3935     PetscCall(DMPlexPermute(dm, perm, &pdm));
3936     PetscCall(ISDestroy(&perm));
3937     PetscCall(DMPlexReplace_Internal(dm, &pdm));
3938     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
3939   }
3940   /* Handle DMPlex distribution */
3941   PetscCall(DMPlexDistributeGetDefault(dm, &distribute));
3942   PetscCall(PetscOptionsBool("-dm_distribute", "Flag to redistribute a mesh among processes", "DMPlexDistribute", distribute, &distribute, NULL));
3943   PetscCall(DMSetFromOptions_Overlap_Plex(dm, PetscOptionsObject, &overlap));
3944   if (distribute) {
3945     DM               pdm = NULL;
3946     PetscPartitioner part;
3947 
3948     PetscCall(DMPlexGetPartitioner(dm, &part));
3949     PetscCall(PetscPartitionerSetFromOptions(part));
3950     PetscCall(DMPlexDistribute(dm, overlap, NULL, &pdm));
3951     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
3952   }
3953   /* Create coordinate space */
3954   if (created) {
3955     DM_Plex  *mesh   = (DM_Plex *)dm->data;
3956     PetscInt  degree = 1;
3957     PetscBool flg;
3958 
3959     PetscCall(PetscOptionsBool("-dm_coord_space", "Use an FEM space for coordinates", "", coordSpace, &coordSpace, &flg));
3960     PetscCall(PetscOptionsInt("-dm_coord_petscspace_degree", "FEM degree for coordinate space", "", degree, &degree, NULL));
3961     if (coordSpace) PetscCall(DMPlexCreateCoordinateSpace(dm, degree, mesh->coordFunc));
3962     if (flg && !coordSpace) {
3963       DM           cdm;
3964       PetscDS      cds;
3965       PetscObject  obj;
3966       PetscClassId id;
3967 
3968       PetscCall(DMGetCoordinateDM(dm, &cdm));
3969       PetscCall(DMGetDS(cdm, &cds));
3970       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
3971       PetscCall(PetscObjectGetClassId(obj, &id));
3972       if (id == PETSCFE_CLASSID) {
3973         PetscContainer dummy;
3974 
3975         PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &dummy));
3976         PetscCall(PetscObjectSetName((PetscObject)dummy, "coordinates"));
3977         PetscCall(DMSetField(cdm, 0, NULL, (PetscObject)dummy));
3978         PetscCall(PetscContainerDestroy(&dummy));
3979         PetscCall(DMClearDS(cdm));
3980       }
3981       mesh->coordFunc = NULL;
3982     }
3983     PetscCall(PetscOptionsBool("-dm_sparse_localize", "Localize only necessary cells", "", dm->sparseLocalize, &dm->sparseLocalize, &flg));
3984     PetscCall(DMLocalizeCoordinates(dm));
3985   }
3986   /* Handle DMPlex refinement */
3987   remap = PETSC_TRUE;
3988   PetscCall(PetscOptionsBoundedInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL, 0));
3989   PetscCall(PetscOptionsBool("-dm_refine_remap", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
3990   PetscCall(PetscOptionsBoundedInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy, 0));
3991   if (refine) PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3992   if (refine && isHierarchy) {
3993     DM *dms, coarseDM;
3994 
3995     PetscCall(DMGetCoarseDM(dm, &coarseDM));
3996     PetscCall(PetscObjectReference((PetscObject)coarseDM));
3997     PetscCall(PetscMalloc1(refine, &dms));
3998     PetscCall(DMRefineHierarchy(dm, refine, dms));
3999     /* Total hack since we do not pass in a pointer */
4000     PetscCall(DMPlexSwap_Static(dm, dms[refine - 1]));
4001     if (refine == 1) {
4002       PetscCall(DMSetCoarseDM(dm, dms[0]));
4003       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4004     } else {
4005       PetscCall(DMSetCoarseDM(dm, dms[refine - 2]));
4006       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4007       PetscCall(DMSetCoarseDM(dms[0], dms[refine - 1]));
4008       PetscCall(DMPlexSetRegularRefinement(dms[0], PETSC_TRUE));
4009     }
4010     PetscCall(DMSetCoarseDM(dms[refine - 1], coarseDM));
4011     PetscCall(PetscObjectDereference((PetscObject)coarseDM));
4012     /* Free DMs */
4013     for (r = 0; r < refine; ++r) {
4014       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4015       PetscCall(DMDestroy(&dms[r]));
4016     }
4017     PetscCall(PetscFree(dms));
4018   } else {
4019     for (r = 0; r < refine; ++r) {
4020       DM             rdm;
4021       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4022 
4023       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4024       PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
4025       /* Total hack since we do not pass in a pointer */
4026       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4027       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4028       if (coordFunc && remap) {
4029         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4030         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4031       }
4032     }
4033   }
4034   /* Handle DMPlex coarsening */
4035   PetscCall(PetscOptionsBoundedInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL, 0));
4036   PetscCall(PetscOptionsBoundedInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy, 0));
4037   if (coarsen && isHierarchy) {
4038     DM *dms;
4039 
4040     PetscCall(PetscMalloc1(coarsen, &dms));
4041     PetscCall(DMCoarsenHierarchy(dm, coarsen, dms));
4042     /* Free DMs */
4043     for (r = 0; r < coarsen; ++r) {
4044       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4045       PetscCall(DMDestroy(&dms[r]));
4046     }
4047     PetscCall(PetscFree(dms));
4048   } else {
4049     for (r = 0; r < coarsen; ++r) {
4050       DM             cdm;
4051       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4052 
4053       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4054       PetscCall(DMCoarsen(dm, PetscObjectComm((PetscObject)dm), &cdm));
4055       /* Total hack since we do not pass in a pointer */
4056       PetscCall(DMPlexReplace_Internal(dm, &cdm));
4057       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4058       if (coordFunc) {
4059         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4060         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4061       }
4062     }
4063   }
4064   /* Handle ghost cells */
4065   PetscCall(PetscOptionsBool("-dm_plex_create_fv_ghost_cells", "Flag to create finite volume ghost cells on the boundary", "DMCreate", ghostCells, &ghostCells, NULL));
4066   if (ghostCells) {
4067     DM   gdm;
4068     char lname[PETSC_MAX_PATH_LEN];
4069 
4070     lname[0] = '\0';
4071     PetscCall(PetscOptionsString("-dm_plex_fv_ghost_cells_label", "Label name for ghost cells boundary", "DMCreate", lname, lname, sizeof(lname), &flg));
4072     PetscCall(DMPlexConstructGhostCells(dm, flg ? lname : NULL, NULL, &gdm));
4073     PetscCall(DMPlexReplace_Internal(dm, &gdm));
4074   }
4075   /* Handle 1D order */
4076   if (reorder != DMPLEX_REORDER_DEFAULT_FALSE && dim == 1) {
4077     DM           cdm, rdm;
4078     PetscDS      cds;
4079     PetscObject  obj;
4080     PetscClassId id = PETSC_OBJECT_CLASSID;
4081     IS           perm;
4082     PetscInt     Nf;
4083     PetscBool    distributed;
4084 
4085     PetscCall(DMPlexIsDistributed(dm, &distributed));
4086     PetscCall(DMGetCoordinateDM(dm, &cdm));
4087     PetscCall(DMGetDS(cdm, &cds));
4088     PetscCall(PetscDSGetNumFields(cds, &Nf));
4089     if (Nf) {
4090       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
4091       PetscCall(PetscObjectGetClassId(obj, &id));
4092     }
4093     if (!distributed && id != PETSCFE_CLASSID) {
4094       PetscCall(DMPlexGetOrdering1D(dm, &perm));
4095       PetscCall(DMPlexPermute(dm, perm, &rdm));
4096       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4097       PetscCall(ISDestroy(&perm));
4098     }
4099   }
4100   /* Handle */
4101   PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4102   PetscOptionsHeadEnd();
4103   PetscFunctionReturn(0);
4104 }
4105 
4106 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm, Vec *vec)
4107 {
4108   PetscFunctionBegin;
4109   PetscCall(DMCreateGlobalVector_Section_Private(dm, vec));
4110   /* PetscCall(VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM)); */
4111   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex));
4112   PetscCall(VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void))VecView_Plex_Native));
4113   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex));
4114   PetscCall(VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void))VecLoad_Plex_Native));
4115   PetscFunctionReturn(0);
4116 }
4117 
4118 static PetscErrorCode DMCreateLocalVector_Plex(DM dm, Vec *vec)
4119 {
4120   PetscFunctionBegin;
4121   PetscCall(DMCreateLocalVector_Section_Private(dm, vec));
4122   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex_Local));
4123   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex_Local));
4124   PetscFunctionReturn(0);
4125 }
4126 
4127 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
4128 {
4129   PetscInt depth, d;
4130 
4131   PetscFunctionBegin;
4132   PetscCall(DMPlexGetDepth(dm, &depth));
4133   if (depth == 1) {
4134     PetscCall(DMGetDimension(dm, &d));
4135     if (dim == 0) PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4136     else if (dim == d) PetscCall(DMPlexGetDepthStratum(dm, 1, pStart, pEnd));
4137     else {
4138       *pStart = 0;
4139       *pEnd   = 0;
4140     }
4141   } else {
4142     PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4143   }
4144   PetscFunctionReturn(0);
4145 }
4146 
4147 static PetscErrorCode DMGetNeighbors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
4148 {
4149   PetscSF            sf;
4150   PetscInt           niranks, njranks, n;
4151   const PetscMPIInt *iranks, *jranks;
4152   DM_Plex           *data = (DM_Plex *)dm->data;
4153 
4154   PetscFunctionBegin;
4155   PetscCall(DMGetPointSF(dm, &sf));
4156   if (!data->neighbors) {
4157     PetscCall(PetscSFSetUp(sf));
4158     PetscCall(PetscSFGetRootRanks(sf, &njranks, &jranks, NULL, NULL, NULL));
4159     PetscCall(PetscSFGetLeafRanks(sf, &niranks, &iranks, NULL, NULL));
4160     PetscCall(PetscMalloc1(njranks + niranks + 1, &data->neighbors));
4161     PetscCall(PetscArraycpy(data->neighbors + 1, jranks, njranks));
4162     PetscCall(PetscArraycpy(data->neighbors + njranks + 1, iranks, niranks));
4163     n = njranks + niranks;
4164     PetscCall(PetscSortRemoveDupsMPIInt(&n, data->neighbors + 1));
4165     /* The following cast should never fail: can't have more neighbors than PETSC_MPI_INT_MAX */
4166     PetscCall(PetscMPIIntCast(n, data->neighbors));
4167   }
4168   if (nranks) *nranks = data->neighbors[0];
4169   if (ranks) {
4170     if (data->neighbors[0]) *ranks = data->neighbors + 1;
4171     else *ranks = NULL;
4172   }
4173   PetscFunctionReturn(0);
4174 }
4175 
4176 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM, DM, Mat, Vec, Vec);
4177 
4178 static PetscErrorCode DMInitialize_Plex(DM dm)
4179 {
4180   PetscFunctionBegin;
4181   dm->ops->view                      = DMView_Plex;
4182   dm->ops->load                      = DMLoad_Plex;
4183   dm->ops->setfromoptions            = DMSetFromOptions_Plex;
4184   dm->ops->clone                     = DMClone_Plex;
4185   dm->ops->setup                     = DMSetUp_Plex;
4186   dm->ops->createlocalsection        = DMCreateLocalSection_Plex;
4187   dm->ops->createdefaultconstraints  = DMCreateDefaultConstraints_Plex;
4188   dm->ops->createglobalvector        = DMCreateGlobalVector_Plex;
4189   dm->ops->createlocalvector         = DMCreateLocalVector_Plex;
4190   dm->ops->getlocaltoglobalmapping   = NULL;
4191   dm->ops->createfieldis             = NULL;
4192   dm->ops->createcoordinatedm        = DMCreateCoordinateDM_Plex;
4193   dm->ops->createcoordinatefield     = DMCreateCoordinateField_Plex;
4194   dm->ops->getcoloring               = NULL;
4195   dm->ops->creatematrix              = DMCreateMatrix_Plex;
4196   dm->ops->createinterpolation       = DMCreateInterpolation_Plex;
4197   dm->ops->createmassmatrix          = DMCreateMassMatrix_Plex;
4198   dm->ops->createmassmatrixlumped    = DMCreateMassMatrixLumped_Plex;
4199   dm->ops->createinjection           = DMCreateInjection_Plex;
4200   dm->ops->refine                    = DMRefine_Plex;
4201   dm->ops->coarsen                   = DMCoarsen_Plex;
4202   dm->ops->refinehierarchy           = DMRefineHierarchy_Plex;
4203   dm->ops->coarsenhierarchy          = DMCoarsenHierarchy_Plex;
4204   dm->ops->extrude                   = DMExtrude_Plex;
4205   dm->ops->globaltolocalbegin        = NULL;
4206   dm->ops->globaltolocalend          = NULL;
4207   dm->ops->localtoglobalbegin        = NULL;
4208   dm->ops->localtoglobalend          = NULL;
4209   dm->ops->destroy                   = DMDestroy_Plex;
4210   dm->ops->createsubdm               = DMCreateSubDM_Plex;
4211   dm->ops->createsuperdm             = DMCreateSuperDM_Plex;
4212   dm->ops->getdimpoints              = DMGetDimPoints_Plex;
4213   dm->ops->locatepoints              = DMLocatePoints_Plex;
4214   dm->ops->projectfunctionlocal      = DMProjectFunctionLocal_Plex;
4215   dm->ops->projectfunctionlabellocal = DMProjectFunctionLabelLocal_Plex;
4216   dm->ops->projectfieldlocal         = DMProjectFieldLocal_Plex;
4217   dm->ops->projectfieldlabellocal    = DMProjectFieldLabelLocal_Plex;
4218   dm->ops->projectbdfieldlabellocal  = DMProjectBdFieldLabelLocal_Plex;
4219   dm->ops->computel2diff             = DMComputeL2Diff_Plex;
4220   dm->ops->computel2gradientdiff     = DMComputeL2GradientDiff_Plex;
4221   dm->ops->computel2fielddiff        = DMComputeL2FieldDiff_Plex;
4222   dm->ops->getneighbors              = DMGetNeighbors_Plex;
4223   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertBoundaryValues_C", DMPlexInsertBoundaryValues_Plex));
4224   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertTimeDerviativeBoundaryValues_C", DMPlexInsertTimeDerivativeBoundaryValues_Plex));
4225   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMSetUpGLVisViewer_C", DMSetUpGLVisViewer_Plex));
4226   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMCreateNeumannOverlap_C", DMCreateNeumannOverlap_Plex));
4227   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4228   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeGetDefault_C", DMPlexDistributeGetDefault_Plex));
4229   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeSetDefault_C", DMPlexDistributeSetDefault_Plex));
4230   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderGetDefault_C", DMPlexReorderGetDefault_Plex));
4231   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderSetDefault_C", DMPlexReorderSetDefault_Plex));
4232   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMInterpolateSolution_C", DMInterpolateSolution_Plex));
4233   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4234   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetOverlap_C", DMPlexSetOverlap_Plex));
4235   PetscFunctionReturn(0);
4236 }
4237 
4238 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
4239 {
4240   DM_Plex *mesh = (DM_Plex *)dm->data;
4241 
4242   PetscFunctionBegin;
4243   mesh->refct++;
4244   (*newdm)->data = mesh;
4245   PetscCall(PetscObjectChangeTypeName((PetscObject)*newdm, DMPLEX));
4246   PetscCall(DMInitialize_Plex(*newdm));
4247   PetscFunctionReturn(0);
4248 }
4249 
4250 /*MC
4251   DMPLEX = "plex" - A DM object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
4252                     In the local representation, Vecs contain all unknowns in the interior and shared boundary. This is
4253                     specified by a PetscSection object. Ownership in the global representation is determined by
4254                     ownership of the underlying DMPlex points. This is specified by another PetscSection object.
4255 
4256   Options Database Keys:
4257 + -dm_refine_pre                     - Refine mesh before distribution
4258 + -dm_refine_uniform_pre             - Choose uniform or generator-based refinement
4259 + -dm_refine_volume_limit_pre        - Cell volume limit after pre-refinement using generator
4260 . -dm_distribute                     - Distribute mesh across processes
4261 . -dm_distribute_overlap             - Number of cells to overlap for distribution
4262 . -dm_refine                         - Refine mesh after distribution
4263 . -dm_plex_hash_location             - Use grid hashing for point location
4264 . -dm_plex_hash_box_faces <n,m,p>    - The number of divisions in each direction of the grid hash
4265 . -dm_plex_partition_balance         - Attempt to evenly divide points on partition boundary between processes
4266 . -dm_plex_remesh_bd                 - Allow changes to the boundary on remeshing
4267 . -dm_plex_max_projection_height     - Maxmimum mesh point height used to project locally
4268 . -dm_plex_regular_refinement        - Use special nested projection algorithm for regular refinement
4269 . -dm_plex_check_all                 - Perform all shecks below
4270 . -dm_plex_check_symmetry            - Check that the adjacency information in the mesh is symmetric
4271 . -dm_plex_check_skeleton <celltype> - Check that each cell has the correct number of vertices
4272 . -dm_plex_check_faces <celltype>    - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
4273 . -dm_plex_check_geometry            - Check that cells have positive volume
4274 . -dm_view :mesh.tex:ascii_latex     - View the mesh in LaTeX/TikZ
4275 . -dm_plex_view_scale <num>          - Scale the TikZ
4276 - -dm_plex_print_fem <num>           - View FEM assembly information, such as element vectors and matrices
4277 
4278   Level: intermediate
4279 
4280 .seealso: `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`
4281 M*/
4282 
4283 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
4284 {
4285   DM_Plex *mesh;
4286   PetscInt unit;
4287 
4288   PetscFunctionBegin;
4289   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4290   PetscCall(PetscNew(&mesh));
4291   dm->data = mesh;
4292 
4293   mesh->refct = 1;
4294   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection));
4295   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection));
4296   mesh->refinementUniform      = PETSC_TRUE;
4297   mesh->refinementLimit        = -1.0;
4298   mesh->distDefault            = PETSC_TRUE;
4299   mesh->reorderDefault         = DMPLEX_REORDER_DEFAULT_NOTSET;
4300   mesh->distributionName       = NULL;
4301   mesh->interpolated           = DMPLEX_INTERPOLATED_INVALID;
4302   mesh->interpolatedCollective = DMPLEX_INTERPOLATED_INVALID;
4303 
4304   PetscCall(PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner));
4305   mesh->remeshBd = PETSC_FALSE;
4306 
4307   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
4308 
4309   mesh->depthState    = -1;
4310   mesh->celltypeState = -1;
4311   mesh->printTol      = 1.0e-10;
4312 
4313   PetscCall(DMInitialize_Plex(dm));
4314   PetscFunctionReturn(0);
4315 }
4316 
4317 /*@
4318   DMPlexCreate - Creates a DMPlex object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
4319 
4320   Collective
4321 
4322   Input Parameter:
4323 . comm - The communicator for the DMPlex object
4324 
4325   Output Parameter:
4326 . mesh  - The DMPlex object
4327 
4328   Level: beginner
4329 
4330 @*/
4331 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
4332 {
4333   PetscFunctionBegin;
4334   PetscValidPointer(mesh, 2);
4335   PetscCall(DMCreate(comm, mesh));
4336   PetscCall(DMSetType(*mesh, DMPLEX));
4337   PetscFunctionReturn(0);
4338 }
4339 
4340 /*@C
4341   DMPlexBuildFromCellListParallel - Build distributed DMPLEX topology from a list of vertices for each cell (common mesh generator output)
4342 
4343   Input Parameters:
4344 + dm - The DM
4345 . numCells - The number of cells owned by this process
4346 . numVertices - The number of vertices to be owned by this process, or PETSC_DECIDE
4347 . NVertices - The global number of vertices, or PETSC_DETERMINE
4348 . numCorners - The number of vertices for each cell
4349 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4350 
4351   Output Parameters:
4352 + vertexSF - (Optional) SF describing complete vertex ownership
4353 - verticesAdjSaved - (Optional) vertex adjacency array
4354 
4355   Notes:
4356   Two triangles sharing a face
4357 $
4358 $        2
4359 $      / | \
4360 $     /  |  \
4361 $    /   |   \
4362 $   0  0 | 1  3
4363 $    \   |   /
4364 $     \  |  /
4365 $      \ | /
4366 $        1
4367 would have input
4368 $  numCells = 2, numVertices = 4
4369 $  cells = [0 1 2  1 3 2]
4370 $
4371 which would result in the DMPlex
4372 $
4373 $        4
4374 $      / | \
4375 $     /  |  \
4376 $    /   |   \
4377 $   2  0 | 1  5
4378 $    \   |   /
4379 $     \  |  /
4380 $      \ | /
4381 $        3
4382 
4383   Vertices are implicitly numbered consecutively 0,...,NVertices.
4384   Each rank owns a chunk of numVertices consecutive vertices.
4385   If numVertices is PETSC_DECIDE, PETSc will distribute them as evenly as possible using PetscLayout.
4386   If NVertices is PETSC_DETERMINE and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
4387   If only NVertices is PETSC_DETERMINE, it is computed as the sum of numVertices over all ranks.
4388 
4389   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
4390 
4391   Not currently supported in Fortran.
4392 
4393   Level: advanced
4394 
4395 .seealso: `DMPlexBuildFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildCoordinatesFromCellListParallel()`
4396 @*/
4397 PetscErrorCode DMPlexBuildFromCellListParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, const PetscInt cells[], PetscSF *vertexSF, PetscInt **verticesAdjSaved)
4398 {
4399   PetscSF     sfPoint;
4400   PetscLayout layout;
4401   PetscInt    numVerticesAdj, *verticesAdj, *cones, c, p;
4402 
4403   PetscFunctionBegin;
4404   PetscValidLogicalCollectiveInt(dm, NVertices, 4);
4405   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4406   /* Get/check global number of vertices */
4407   {
4408     PetscInt       NVerticesInCells, i;
4409     const PetscInt len = numCells * numCorners;
4410 
4411     /* NVerticesInCells = max(cells) + 1 */
4412     NVerticesInCells = PETSC_MIN_INT;
4413     for (i = 0; i < len; i++)
4414       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4415     ++NVerticesInCells;
4416     PetscCallMPI(MPI_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
4417 
4418     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
4419     else
4420       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
4421   }
4422   /* Count locally unique vertices */
4423   {
4424     PetscHSetI vhash;
4425     PetscInt   off = 0;
4426 
4427     PetscCall(PetscHSetICreate(&vhash));
4428     for (c = 0; c < numCells; ++c) {
4429       for (p = 0; p < numCorners; ++p) PetscCall(PetscHSetIAdd(vhash, cells[c * numCorners + p]));
4430     }
4431     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
4432     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
4433     else verticesAdj = *verticesAdjSaved;
4434     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
4435     PetscCall(PetscHSetIDestroy(&vhash));
4436     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
4437   }
4438   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
4439   /* Create cones */
4440   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
4441   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
4442   PetscCall(DMSetUp(dm));
4443   PetscCall(DMPlexGetCones(dm, &cones));
4444   for (c = 0; c < numCells; ++c) {
4445     for (p = 0; p < numCorners; ++p) {
4446       const PetscInt gv = cells[c * numCorners + p];
4447       PetscInt       lv;
4448 
4449       /* Positions within verticesAdj form 0-based local vertex numbering;
4450          we need to shift it by numCells to get correct DAG points (cells go first) */
4451       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
4452       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
4453       cones[c * numCorners + p] = lv + numCells;
4454     }
4455   }
4456   /* Build point sf */
4457   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
4458   PetscCall(PetscLayoutSetSize(layout, NVertices));
4459   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
4460   PetscCall(PetscLayoutSetBlockSize(layout, 1));
4461   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
4462   PetscCall(PetscLayoutDestroy(&layout));
4463   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
4464   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
4465   if (dm->sf) {
4466     const char *prefix;
4467 
4468     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
4469     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
4470   }
4471   PetscCall(DMSetPointSF(dm, sfPoint));
4472   PetscCall(PetscSFDestroy(&sfPoint));
4473   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)(*vertexSF), "Vertex Ownership SF"));
4474   /* Fill in the rest of the topology structure */
4475   PetscCall(DMPlexSymmetrize(dm));
4476   PetscCall(DMPlexStratify(dm));
4477   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4478   PetscFunctionReturn(0);
4479 }
4480 
4481 /*@C
4482   DMPlexBuildCoordinatesFromCellListParallel - Build DM coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4483 
4484   Input Parameters:
4485 + dm - The DM
4486 . spaceDim - The spatial dimension used for coordinates
4487 . sfVert - SF describing complete vertex ownership
4488 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4489 
4490   Level: advanced
4491 
4492   Notes:
4493   Not currently supported in Fortran.
4494 
4495 .seealso: `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellListParallel()`
4496 @*/
4497 PetscErrorCode DMPlexBuildCoordinatesFromCellListParallel(DM dm, PetscInt spaceDim, PetscSF sfVert, const PetscReal vertexCoords[])
4498 {
4499   PetscSection coordSection;
4500   Vec          coordinates;
4501   PetscScalar *coords;
4502   PetscInt     numVertices, numVerticesAdj, coordSize, v, vStart, vEnd;
4503 
4504   PetscFunctionBegin;
4505   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4506   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
4507   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4508   PetscCall(DMSetCoordinateDim(dm, spaceDim));
4509   PetscCall(PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL));
4510   PetscCheck(vEnd - vStart == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Supplied sfVert has wrong number of leaves = %" PetscInt_FMT " != %" PetscInt_FMT " = vEnd - vStart", numVerticesAdj, vEnd - vStart);
4511   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4512   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4513   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
4514   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
4515   for (v = vStart; v < vEnd; ++v) {
4516     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
4517     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
4518   }
4519   PetscCall(PetscSectionSetUp(coordSection));
4520   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4521   PetscCall(VecCreate(PetscObjectComm((PetscObject)dm), &coordinates));
4522   PetscCall(VecSetBlockSize(coordinates, spaceDim));
4523   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4524   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4525   PetscCall(VecSetType(coordinates, VECSTANDARD));
4526   PetscCall(VecGetArray(coordinates, &coords));
4527   {
4528     MPI_Datatype coordtype;
4529 
4530     /* Need a temp buffer for coords if we have complex/single */
4531     PetscCallMPI(MPI_Type_contiguous(spaceDim, MPIU_SCALAR, &coordtype));
4532     PetscCallMPI(MPI_Type_commit(&coordtype));
4533 #if defined(PETSC_USE_COMPLEX)
4534     {
4535       PetscScalar *svertexCoords;
4536       PetscInt     i;
4537       PetscCall(PetscMalloc1(numVertices * spaceDim, &svertexCoords));
4538       for (i = 0; i < numVertices * spaceDim; i++) svertexCoords[i] = vertexCoords[i];
4539       PetscCall(PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4540       PetscCall(PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4541       PetscCall(PetscFree(svertexCoords));
4542     }
4543 #else
4544     PetscCall(PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4545     PetscCall(PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4546 #endif
4547     PetscCallMPI(MPI_Type_free(&coordtype));
4548   }
4549   PetscCall(VecRestoreArray(coordinates, &coords));
4550   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4551   PetscCall(VecDestroy(&coordinates));
4552   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4553   PetscFunctionReturn(0);
4554 }
4555 
4556 /*@
4557   DMPlexCreateFromCellListParallelPetsc - Create distributed DMPLEX from a list of vertices for each cell (common mesh generator output)
4558 
4559   Input Parameters:
4560 + comm - The communicator
4561 . dim - The topological dimension of the mesh
4562 . numCells - The number of cells owned by this process
4563 . numVertices - The number of vertices owned by this process, or PETSC_DECIDE
4564 . NVertices - The global number of vertices, or PETSC_DECIDE
4565 . numCorners - The number of vertices for each cell
4566 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4567 . cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4568 . spaceDim - The spatial dimension used for coordinates
4569 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4570 
4571   Output Parameters:
4572 + dm - The DM
4573 . vertexSF - (Optional) SF describing complete vertex ownership
4574 - verticesAdjSaved - (Optional) vertex adjacency array
4575 
4576   Notes:
4577   This function is just a convenient sequence of DMCreate(), DMSetType(), DMSetDimension(),
4578   DMPlexBuildFromCellListParallel(), DMPlexInterpolate(), DMPlexBuildCoordinatesFromCellListParallel()
4579 
4580   See DMPlexBuildFromCellListParallel() for an example and details about the topology-related parameters.
4581   See DMPlexBuildCoordinatesFromCellListParallel() for details about the geometry-related parameters.
4582 
4583   Level: intermediate
4584 
4585 .seealso: `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
4586 @*/
4587 PetscErrorCode DMPlexCreateFromCellListParallelPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, PetscInt **verticesAdj, DM *dm)
4588 {
4589   PetscSF sfVert;
4590 
4591   PetscFunctionBegin;
4592   PetscCall(DMCreate(comm, dm));
4593   PetscCall(DMSetType(*dm, DMPLEX));
4594   PetscValidLogicalCollectiveInt(*dm, dim, 2);
4595   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
4596   PetscCall(DMSetDimension(*dm, dim));
4597   PetscCall(DMPlexBuildFromCellListParallel(*dm, numCells, numVertices, NVertices, numCorners, cells, &sfVert, verticesAdj));
4598   if (interpolate) {
4599     DM idm;
4600 
4601     PetscCall(DMPlexInterpolate(*dm, &idm));
4602     PetscCall(DMDestroy(dm));
4603     *dm = idm;
4604   }
4605   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
4606   if (vertexSF) *vertexSF = sfVert;
4607   else PetscCall(PetscSFDestroy(&sfVert));
4608   PetscFunctionReturn(0);
4609 }
4610 
4611 /*@C
4612   DMPlexBuildFromCellList - Build DMPLEX topology from a list of vertices for each cell (common mesh generator output)
4613 
4614   Input Parameters:
4615 + dm - The DM
4616 . numCells - The number of cells owned by this process
4617 . numVertices - The number of vertices owned by this process, or PETSC_DETERMINE
4618 . numCorners - The number of vertices for each cell
4619 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4620 
4621   Level: advanced
4622 
4623   Notes:
4624   Two triangles sharing a face
4625 $
4626 $        2
4627 $      / | \
4628 $     /  |  \
4629 $    /   |   \
4630 $   0  0 | 1  3
4631 $    \   |   /
4632 $     \  |  /
4633 $      \ | /
4634 $        1
4635 would have input
4636 $  numCells = 2, numVertices = 4
4637 $  cells = [0 1 2  1 3 2]
4638 $
4639 which would result in the DMPlex
4640 $
4641 $        4
4642 $      / | \
4643 $     /  |  \
4644 $    /   |   \
4645 $   2  0 | 1  5
4646 $    \   |   /
4647 $     \  |  /
4648 $      \ | /
4649 $        3
4650 
4651   If numVertices is PETSC_DETERMINE, it is computed by PETSc as the maximum vertex index in cells + 1.
4652 
4653   Not currently supported in Fortran.
4654 
4655 .seealso: `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListPetsc()`
4656 @*/
4657 PetscErrorCode DMPlexBuildFromCellList(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const PetscInt cells[])
4658 {
4659   PetscInt *cones, c, p, dim;
4660 
4661   PetscFunctionBegin;
4662   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4663   PetscCall(DMGetDimension(dm, &dim));
4664   /* Get/check global number of vertices */
4665   {
4666     PetscInt       NVerticesInCells, i;
4667     const PetscInt len = numCells * numCorners;
4668 
4669     /* NVerticesInCells = max(cells) + 1 */
4670     NVerticesInCells = PETSC_MIN_INT;
4671     for (i = 0; i < len; i++)
4672       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4673     ++NVerticesInCells;
4674 
4675     if (numVertices == PETSC_DECIDE) numVertices = NVerticesInCells;
4676     else
4677       PetscCheck(numVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, numVertices, NVerticesInCells);
4678   }
4679   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
4680   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
4681   PetscCall(DMSetUp(dm));
4682   PetscCall(DMPlexGetCones(dm, &cones));
4683   for (c = 0; c < numCells; ++c) {
4684     for (p = 0; p < numCorners; ++p) cones[c * numCorners + p] = cells[c * numCorners + p] + numCells;
4685   }
4686   PetscCall(DMPlexSymmetrize(dm));
4687   PetscCall(DMPlexStratify(dm));
4688   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4689   PetscFunctionReturn(0);
4690 }
4691 
4692 /*@C
4693   DMPlexBuildCoordinatesFromCellList - Build DM coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4694 
4695   Input Parameters:
4696 + dm - The DM
4697 . spaceDim - The spatial dimension used for coordinates
4698 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4699 
4700   Level: advanced
4701 
4702   Notes:
4703   Not currently supported in Fortran.
4704 
4705 .seealso: `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellList()`
4706 @*/
4707 PetscErrorCode DMPlexBuildCoordinatesFromCellList(DM dm, PetscInt spaceDim, const PetscReal vertexCoords[])
4708 {
4709   PetscSection coordSection;
4710   Vec          coordinates;
4711   DM           cdm;
4712   PetscScalar *coords;
4713   PetscInt     v, vStart, vEnd, d;
4714 
4715   PetscFunctionBegin;
4716   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4717   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
4718   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4719   PetscCall(DMSetCoordinateDim(dm, spaceDim));
4720   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4721   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4722   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
4723   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
4724   for (v = vStart; v < vEnd; ++v) {
4725     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
4726     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
4727   }
4728   PetscCall(PetscSectionSetUp(coordSection));
4729 
4730   PetscCall(DMGetCoordinateDM(dm, &cdm));
4731   PetscCall(DMCreateLocalVector(cdm, &coordinates));
4732   PetscCall(VecSetBlockSize(coordinates, spaceDim));
4733   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4734   PetscCall(VecGetArrayWrite(coordinates, &coords));
4735   for (v = 0; v < vEnd - vStart; ++v) {
4736     for (d = 0; d < spaceDim; ++d) coords[v * spaceDim + d] = vertexCoords[v * spaceDim + d];
4737   }
4738   PetscCall(VecRestoreArrayWrite(coordinates, &coords));
4739   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4740   PetscCall(VecDestroy(&coordinates));
4741   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4742   PetscFunctionReturn(0);
4743 }
4744 
4745 /*@
4746   DMPlexCreateFromCellListPetsc - Create DMPLEX from a list of vertices for each cell (common mesh generator output), but only process 0 takes in the input
4747 
4748   Collective on comm
4749 
4750   Input Parameters:
4751 + comm - The communicator
4752 . dim - The topological dimension of the mesh
4753 . numCells - The number of cells, only on process 0
4754 . numVertices - The number of vertices owned by this process, or PETSC_DECIDE, only on process 0
4755 . numCorners - The number of vertices for each cell, only on process 0
4756 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4757 . cells - An array of numCells*numCorners numbers, the vertices for each cell, only on process 0
4758 . spaceDim - The spatial dimension used for coordinates
4759 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex, only on process 0
4760 
4761   Output Parameter:
4762 . dm - The DM, which only has points on process 0
4763 
4764   Notes:
4765   This function is just a convenient sequence of DMCreate(), DMSetType(), DMSetDimension(), DMPlexBuildFromCellList(),
4766   DMPlexInterpolate(), DMPlexBuildCoordinatesFromCellList()
4767 
4768   See DMPlexBuildFromCellList() for an example and details about the topology-related parameters.
4769   See DMPlexBuildCoordinatesFromCellList() for details about the geometry-related parameters.
4770   See DMPlexCreateFromCellListParallelPetsc() for parallel input
4771 
4772   Level: intermediate
4773 
4774 .seealso: `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellList()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
4775 @*/
4776 PetscErrorCode DMPlexCreateFromCellListPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], DM *dm)
4777 {
4778   PetscMPIInt rank;
4779 
4780   PetscFunctionBegin;
4781   PetscCheck(dim, comm, PETSC_ERR_ARG_OUTOFRANGE, "This is not appropriate for 0-dimensional meshes. Consider either creating the DM using DMPlexCreateFromDAG(), by hand, or using DMSwarm.");
4782   PetscCallMPI(MPI_Comm_rank(comm, &rank));
4783   PetscCall(DMCreate(comm, dm));
4784   PetscCall(DMSetType(*dm, DMPLEX));
4785   PetscCall(DMSetDimension(*dm, dim));
4786   if (rank == 0) PetscCall(DMPlexBuildFromCellList(*dm, numCells, numVertices, numCorners, cells));
4787   else PetscCall(DMPlexBuildFromCellList(*dm, 0, 0, 0, NULL));
4788   if (interpolate) {
4789     DM idm;
4790 
4791     PetscCall(DMPlexInterpolate(*dm, &idm));
4792     PetscCall(DMDestroy(dm));
4793     *dm = idm;
4794   }
4795   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, vertexCoords));
4796   else PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, NULL));
4797   PetscFunctionReturn(0);
4798 }
4799 
4800 /*@
4801   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a DM
4802 
4803   Input Parameters:
4804 + dm - The empty DM object, usually from DMCreate() and DMSetDimension()
4805 . depth - The depth of the DAG
4806 . numPoints - Array of size depth + 1 containing the number of points at each depth
4807 . coneSize - The cone size of each point
4808 . cones - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
4809 . coneOrientations - The orientation of each cone point
4810 - vertexCoords - An array of numPoints[0]*spacedim numbers representing the coordinates of each vertex, with spacedim the value set via DMSetCoordinateDim()
4811 
4812   Output Parameter:
4813 . dm - The DM
4814 
4815   Note: Two triangles sharing a face would have input
4816 $  depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
4817 $  cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
4818 $ vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
4819 $
4820 which would result in the DMPlex
4821 $
4822 $        4
4823 $      / | \
4824 $     /  |  \
4825 $    /   |   \
4826 $   2  0 | 1  5
4827 $    \   |   /
4828 $     \  |  /
4829 $      \ | /
4830 $        3
4831 $
4832 $ Notice that all points are numbered consecutively, unlike DMPlexCreateFromCellListPetsc()
4833 
4834   Level: advanced
4835 
4836 .seealso: `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
4837 @*/
4838 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
4839 {
4840   Vec          coordinates;
4841   PetscSection coordSection;
4842   PetscScalar *coords;
4843   PetscInt     coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
4844 
4845   PetscFunctionBegin;
4846   PetscCall(DMGetDimension(dm, &dim));
4847   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
4848   PetscCheck(dimEmbed >= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Embedding dimension %" PetscInt_FMT " cannot be less than intrinsic dimension %" PetscInt_FMT, dimEmbed, dim);
4849   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
4850   PetscCall(DMPlexSetChart(dm, pStart, pEnd));
4851   for (p = pStart; p < pEnd; ++p) {
4852     PetscCall(DMPlexSetConeSize(dm, p, coneSize[p - pStart]));
4853     if (firstVertex < 0 && !coneSize[p - pStart]) firstVertex = p - pStart;
4854   }
4855   PetscCheck(firstVertex >= 0 || !numPoints[0], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Expected %" PetscInt_FMT " vertices but could not find any", numPoints[0]);
4856   PetscCall(DMSetUp(dm)); /* Allocate space for cones */
4857   for (p = pStart, off = 0; p < pEnd; off += coneSize[p - pStart], ++p) {
4858     PetscCall(DMPlexSetCone(dm, p, &cones[off]));
4859     PetscCall(DMPlexSetConeOrientation(dm, p, &coneOrientations[off]));
4860   }
4861   PetscCall(DMPlexSymmetrize(dm));
4862   PetscCall(DMPlexStratify(dm));
4863   /* Build coordinates */
4864   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4865   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4866   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dimEmbed));
4867   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numPoints[0]));
4868   for (v = firstVertex; v < firstVertex + numPoints[0]; ++v) {
4869     PetscCall(PetscSectionSetDof(coordSection, v, dimEmbed));
4870     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed));
4871   }
4872   PetscCall(PetscSectionSetUp(coordSection));
4873   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4874   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
4875   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4876   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4877   PetscCall(VecSetBlockSize(coordinates, dimEmbed));
4878   PetscCall(VecSetType(coordinates, VECSTANDARD));
4879   if (vertexCoords) {
4880     PetscCall(VecGetArray(coordinates, &coords));
4881     for (v = 0; v < numPoints[0]; ++v) {
4882       PetscInt off;
4883 
4884       PetscCall(PetscSectionGetOffset(coordSection, v + firstVertex, &off));
4885       for (d = 0; d < dimEmbed; ++d) coords[off + d] = vertexCoords[v * dimEmbed + d];
4886     }
4887   }
4888   PetscCall(VecRestoreArray(coordinates, &coords));
4889   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4890   PetscCall(VecDestroy(&coordinates));
4891   PetscFunctionReturn(0);
4892 }
4893 
4894 /*@C
4895   DMPlexCreateCellVertexFromFile - Create a DMPlex mesh from a simple cell-vertex file.
4896 
4897 + comm        - The MPI communicator
4898 . filename    - Name of the .dat file
4899 - interpolate - Create faces and edges in the mesh
4900 
4901   Output Parameter:
4902 . dm  - The DM object representing the mesh
4903 
4904   Note: The format is the simplest possible:
4905 $ Ne
4906 $ v0 v1 ... vk
4907 $ Nv
4908 $ x y z marker
4909 
4910   Level: beginner
4911 
4912 .seealso: `DMPlexCreateFromFile()`, `DMPlexCreateMedFromFile()`, `DMPlexCreateGmsh()`, `DMPlexCreate()`
4913 @*/
4914 PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
4915 {
4916   DMLabel      marker;
4917   PetscViewer  viewer;
4918   Vec          coordinates;
4919   PetscSection coordSection;
4920   PetscScalar *coords;
4921   char         line[PETSC_MAX_PATH_LEN];
4922   PetscInt     dim = 3, cdim = 3, coordSize, v, c, d;
4923   PetscMPIInt  rank;
4924   int          snum, Nv, Nc, Ncn, Nl;
4925 
4926   PetscFunctionBegin;
4927   PetscCallMPI(MPI_Comm_rank(comm, &rank));
4928   PetscCall(PetscViewerCreate(comm, &viewer));
4929   PetscCall(PetscViewerSetType(viewer, PETSCVIEWERASCII));
4930   PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
4931   PetscCall(PetscViewerFileSetName(viewer, filename));
4932   if (rank == 0) {
4933     PetscCall(PetscViewerRead(viewer, line, 4, NULL, PETSC_STRING));
4934     snum = sscanf(line, "%d %d %d %d", &Nc, &Nv, &Ncn, &Nl);
4935     PetscCheck(snum == 4, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4936   } else {
4937     Nc = Nv = Ncn = Nl = 0;
4938   }
4939   PetscCall(DMCreate(comm, dm));
4940   PetscCall(DMSetType(*dm, DMPLEX));
4941   PetscCall(DMPlexSetChart(*dm, 0, Nc + Nv));
4942   PetscCall(DMSetDimension(*dm, dim));
4943   PetscCall(DMSetCoordinateDim(*dm, cdim));
4944   /* Read topology */
4945   if (rank == 0) {
4946     char     format[PETSC_MAX_PATH_LEN];
4947     PetscInt cone[8];
4948     int      vbuf[8], v;
4949 
4950     for (c = 0; c < Ncn; ++c) {
4951       format[c * 3 + 0] = '%';
4952       format[c * 3 + 1] = 'd';
4953       format[c * 3 + 2] = ' ';
4954     }
4955     format[Ncn * 3 - 1] = '\0';
4956     for (c = 0; c < Nc; ++c) PetscCall(DMPlexSetConeSize(*dm, c, Ncn));
4957     PetscCall(DMSetUp(*dm));
4958     for (c = 0; c < Nc; ++c) {
4959       PetscCall(PetscViewerRead(viewer, line, Ncn, NULL, PETSC_STRING));
4960       switch (Ncn) {
4961       case 2:
4962         snum = sscanf(line, format, &vbuf[0], &vbuf[1]);
4963         break;
4964       case 3:
4965         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2]);
4966         break;
4967       case 4:
4968         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3]);
4969         break;
4970       case 6:
4971         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5]);
4972         break;
4973       case 8:
4974         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);
4975         break;
4976       default:
4977         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No cell shape with %d vertices", Ncn);
4978       }
4979       PetscCheck(snum == Ncn, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
4980       for (v = 0; v < Ncn; ++v) cone[v] = vbuf[v] + Nc;
4981       /* Hexahedra are inverted */
4982       if (Ncn == 8) {
4983         PetscInt tmp = cone[1];
4984         cone[1]      = cone[3];
4985         cone[3]      = tmp;
4986       }
4987       PetscCall(DMPlexSetCone(*dm, c, cone));
4988     }
4989   }
4990   PetscCall(DMPlexSymmetrize(*dm));
4991   PetscCall(DMPlexStratify(*dm));
4992   /* Read coordinates */
4993   PetscCall(DMGetCoordinateSection(*dm, &coordSection));
4994   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4995   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
4996   PetscCall(PetscSectionSetChart(coordSection, Nc, Nc + Nv));
4997   for (v = Nc; v < Nc + Nv; ++v) {
4998     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
4999     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
5000   }
5001   PetscCall(PetscSectionSetUp(coordSection));
5002   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
5003   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
5004   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5005   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
5006   PetscCall(VecSetBlockSize(coordinates, cdim));
5007   PetscCall(VecSetType(coordinates, VECSTANDARD));
5008   PetscCall(VecGetArray(coordinates, &coords));
5009   if (rank == 0) {
5010     char   format[PETSC_MAX_PATH_LEN];
5011     double x[3];
5012     int    l, val[3];
5013 
5014     if (Nl) {
5015       for (l = 0; l < Nl; ++l) {
5016         format[l * 3 + 0] = '%';
5017         format[l * 3 + 1] = 'd';
5018         format[l * 3 + 2] = ' ';
5019       }
5020       format[Nl * 3 - 1] = '\0';
5021       PetscCall(DMCreateLabel(*dm, "marker"));
5022       PetscCall(DMGetLabel(*dm, "marker", &marker));
5023     }
5024     for (v = 0; v < Nv; ++v) {
5025       PetscCall(PetscViewerRead(viewer, line, 3 + Nl, NULL, PETSC_STRING));
5026       snum = sscanf(line, "%lg %lg %lg", &x[0], &x[1], &x[2]);
5027       PetscCheck(snum == 3, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5028       switch (Nl) {
5029       case 0:
5030         snum = 0;
5031         break;
5032       case 1:
5033         snum = sscanf(line, format, &val[0]);
5034         break;
5035       case 2:
5036         snum = sscanf(line, format, &val[0], &val[1]);
5037         break;
5038       case 3:
5039         snum = sscanf(line, format, &val[0], &val[1], &val[2]);
5040         break;
5041       default:
5042         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Request support for %d labels", Nl);
5043       }
5044       PetscCheck(snum == Nl, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5045       for (d = 0; d < cdim; ++d) coords[v * cdim + d] = x[d];
5046       for (l = 0; l < Nl; ++l) PetscCall(DMLabelSetValue(marker, v + Nc, val[l]));
5047     }
5048   }
5049   PetscCall(VecRestoreArray(coordinates, &coords));
5050   PetscCall(DMSetCoordinatesLocal(*dm, coordinates));
5051   PetscCall(VecDestroy(&coordinates));
5052   PetscCall(PetscViewerDestroy(&viewer));
5053   if (interpolate) {
5054     DM      idm;
5055     DMLabel bdlabel;
5056 
5057     PetscCall(DMPlexInterpolate(*dm, &idm));
5058     PetscCall(DMDestroy(dm));
5059     *dm = idm;
5060 
5061     if (!Nl) {
5062       PetscCall(DMCreateLabel(*dm, "marker"));
5063       PetscCall(DMGetLabel(*dm, "marker", &bdlabel));
5064       PetscCall(DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel));
5065       PetscCall(DMPlexLabelComplete(*dm, bdlabel));
5066     }
5067   }
5068   PetscFunctionReturn(0);
5069 }
5070 
5071 /*@C
5072   DMPlexCreateFromFile - This takes a filename and produces a DM
5073 
5074   Input Parameters:
5075 + comm - The communicator
5076 . filename - A file name
5077 . plexname - The object name of the resulting DM, also used for intra-datafile lookup by some formats
5078 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
5079 
5080   Output Parameter:
5081 . dm - The DM
5082 
5083   Options Database Keys:
5084 . -dm_plex_create_from_hdf5_xdmf - use the PETSC_VIEWER_HDF5_XDMF format for reading HDF5
5085 
5086   Use -dm_plex_create_ prefix to pass options to the internal PetscViewer, e.g.
5087 $ -dm_plex_create_viewer_hdf5_collective
5088 
5089   Notes:
5090   Using PETSCVIEWERHDF5 type with PETSC_VIEWER_HDF5_PETSC format, one can save multiple DMPlex
5091   meshes in a single HDF5 file. This in turn requires one to name the DMPlex object with PetscObjectSetName()
5092   before saving it with DMView() and before loading it with DMLoad() for identification of the mesh object.
5093   The input parameter name is thus used to name the DMPlex object when DMPlexCreateFromFile() internally
5094   calls DMLoad(). Currently, name is ignored for other viewer types and/or formats.
5095 
5096   Level: beginner
5097 
5098 .seealso: `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`, `PetscObjectSetName()`, `DMView()`, `DMLoad()`
5099 @*/
5100 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], const char plexname[], PetscBool interpolate, DM *dm)
5101 {
5102   const char  extGmsh[]      = ".msh";
5103   const char  extGmsh2[]     = ".msh2";
5104   const char  extGmsh4[]     = ".msh4";
5105   const char  extCGNS[]      = ".cgns";
5106   const char  extExodus[]    = ".exo";
5107   const char  extExodus_e[]  = ".e";
5108   const char  extGenesis[]   = ".gen";
5109   const char  extFluent[]    = ".cas";
5110   const char  extHDF5[]      = ".h5";
5111   const char  extMed[]       = ".med";
5112   const char  extPLY[]       = ".ply";
5113   const char  extEGADSLite[] = ".egadslite";
5114   const char  extEGADS[]     = ".egads";
5115   const char  extIGES[]      = ".igs";
5116   const char  extSTEP[]      = ".stp";
5117   const char  extCV[]        = ".dat";
5118   size_t      len;
5119   PetscBool   isGmsh, isGmsh2, isGmsh4, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isMed, isPLY, isEGADSLite, isEGADS, isIGES, isSTEP, isCV;
5120   PetscMPIInt rank;
5121 
5122   PetscFunctionBegin;
5123   PetscValidCharPointer(filename, 2);
5124   if (plexname) PetscValidCharPointer(plexname, 3);
5125   PetscValidPointer(dm, 5);
5126   PetscCall(DMInitializePackage());
5127   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5128   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5129   PetscCall(PetscStrlen(filename, &len));
5130   PetscCheck(len, comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
5131 
5132 #define CheckExtension(extension__, is_extension__) \
5133   do { \
5134     PetscAssert(sizeof(extension__), comm, PETSC_ERR_PLIB, "Zero-size extension: %s", extension__); \
5135     /* don't count the null-terminator at the end */ \
5136     const size_t ext_len = sizeof(extension__) - 1; \
5137     if (len < ext_len) { \
5138       is_extension__ = PETSC_FALSE; \
5139     } else { \
5140       PetscCall(PetscStrncmp(filename + len - ext_len, extension__, ext_len, &is_extension__)); \
5141     } \
5142   } while (0)
5143 
5144   CheckExtension(extGmsh, isGmsh);
5145   CheckExtension(extGmsh2, isGmsh2);
5146   CheckExtension(extGmsh4, isGmsh4);
5147   CheckExtension(extCGNS, isCGNS);
5148   CheckExtension(extExodus, isExodus);
5149   if (!isExodus) CheckExtension(extExodus_e, isExodus);
5150   CheckExtension(extGenesis, isGenesis);
5151   CheckExtension(extFluent, isFluent);
5152   CheckExtension(extHDF5, isHDF5);
5153   CheckExtension(extMed, isMed);
5154   CheckExtension(extPLY, isPLY);
5155   CheckExtension(extEGADSLite, isEGADSLite);
5156   CheckExtension(extEGADS, isEGADS);
5157   CheckExtension(extIGES, isIGES);
5158   CheckExtension(extSTEP, isSTEP);
5159   CheckExtension(extCV, isCV);
5160 
5161 #undef CheckExtension
5162 
5163   if (isGmsh || isGmsh2 || isGmsh4) {
5164     PetscCall(DMPlexCreateGmshFromFile(comm, filename, interpolate, dm));
5165   } else if (isCGNS) {
5166     PetscCall(DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm));
5167   } else if (isExodus || isGenesis) {
5168     PetscCall(DMPlexCreateExodusFromFile(comm, filename, interpolate, dm));
5169   } else if (isFluent) {
5170     PetscCall(DMPlexCreateFluentFromFile(comm, filename, interpolate, dm));
5171   } else if (isHDF5) {
5172     PetscBool   load_hdf5_xdmf = PETSC_FALSE;
5173     PetscViewer viewer;
5174 
5175     /* PETSC_VIEWER_HDF5_XDMF is used if the filename ends with .xdmf.h5, or if -dm_plex_create_from_hdf5_xdmf option is present */
5176     PetscCall(PetscStrncmp(&filename[PetscMax(0, len - 8)], ".xdmf", 5, &load_hdf5_xdmf));
5177     PetscCall(PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &load_hdf5_xdmf, NULL));
5178     PetscCall(PetscViewerCreate(comm, &viewer));
5179     PetscCall(PetscViewerSetType(viewer, PETSCVIEWERHDF5));
5180     PetscCall(PetscViewerSetOptionsPrefix(viewer, "dm_plex_create_"));
5181     PetscCall(PetscViewerSetFromOptions(viewer));
5182     PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
5183     PetscCall(PetscViewerFileSetName(viewer, filename));
5184 
5185     PetscCall(DMCreate(comm, dm));
5186     PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5187     PetscCall(DMSetType(*dm, DMPLEX));
5188     if (load_hdf5_xdmf) PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF));
5189     PetscCall(DMLoad(*dm, viewer));
5190     if (load_hdf5_xdmf) PetscCall(PetscViewerPopFormat(viewer));
5191     PetscCall(PetscViewerDestroy(&viewer));
5192 
5193     if (interpolate) {
5194       DM idm;
5195 
5196       PetscCall(DMPlexInterpolate(*dm, &idm));
5197       PetscCall(DMDestroy(dm));
5198       *dm = idm;
5199     }
5200   } else if (isMed) {
5201     PetscCall(DMPlexCreateMedFromFile(comm, filename, interpolate, dm));
5202   } else if (isPLY) {
5203     PetscCall(DMPlexCreatePLYFromFile(comm, filename, interpolate, dm));
5204   } else if (isEGADSLite || isEGADS || isIGES || isSTEP) {
5205     if (isEGADSLite) PetscCall(DMPlexCreateEGADSLiteFromFile(comm, filename, dm));
5206     else PetscCall(DMPlexCreateEGADSFromFile(comm, filename, dm));
5207     if (!interpolate) {
5208       DM udm;
5209 
5210       PetscCall(DMPlexUninterpolate(*dm, &udm));
5211       PetscCall(DMDestroy(dm));
5212       *dm = udm;
5213     }
5214   } else if (isCV) {
5215     PetscCall(DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm));
5216   } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
5217   PetscCall(PetscStrlen(plexname, &len));
5218   if (len) PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5219   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5220   PetscFunctionReturn(0);
5221 }
5222 /*@C
5223   DMPlexCreateEphemeral - This takes a `DMPlexTransform` and a base `DMPlex` and produces an ephemeral `DM`, meaning one that is created on the fly in response to queries.
5224 
5225   Input Parameter:
5226 . tr - The `DMPlexTransform`
5227 
5228   Output Parameter:
5229 . dm - The `DM`
5230 
5231   Notes:
5232   An emphemeral mesh is one that is not stored concretely, as in the default Plex implementation, but rather is produced on the fly in response to queries, using information from the transform and the base mesh.
5233 
5234   Level: beginner
5235 
5236 .seealso: `DMPlexCreateFromFile`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
5237 @*/
5238 PetscErrorCode DMPlexCreateEphemeral(DMPlexTransform tr, DM *dm)
5239 {
5240   DM       bdm;
5241   PetscInt Nl;
5242 
5243   PetscFunctionBegin;
5244   PetscCall(DMCreate(PetscObjectComm((PetscObject)tr), dm));
5245   PetscCall(DMSetType(*dm, DMPLEX));
5246   PetscCall(DMSetFromOptions(*dm));
5247 
5248   PetscCall(PetscObjectReference((PetscObject)tr));
5249   PetscCall(DMPlexTransformDestroy(&((DM_Plex *)(*dm)->data)->tr));
5250   ((DM_Plex *)(*dm)->data)->tr = tr;
5251 
5252   PetscCall(DMPlexTransformGetDM(tr, &bdm));
5253   PetscCall(DMGetNumLabels(bdm, &Nl));
5254   for (PetscInt l = 0; l < Nl; ++l) {
5255     DMLabel     label, labelNew;
5256     const char *lname;
5257     PetscBool   isDepth, isCellType;
5258 
5259     PetscCall(DMGetLabelName(bdm, l, &lname));
5260     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
5261     if (isDepth) continue;
5262     PetscCall(PetscStrcmp(lname, "celltype", &isCellType));
5263     if (isCellType) continue;
5264     PetscCall(DMCreateLabel(*dm, lname));
5265     PetscCall(DMGetLabel(bdm, lname, &label));
5266     PetscCall(DMGetLabel(*dm, lname, &labelNew));
5267     PetscCall(DMLabelSetType(labelNew, DMLABELEPHEMERAL));
5268     PetscCall(DMLabelEphemeralSetLabel(labelNew, label));
5269     PetscCall(DMLabelEphemeralSetTransform(labelNew, tr));
5270     PetscCall(DMLabelSetUp(labelNew));
5271   }
5272   PetscFunctionReturn(0);
5273 }
5274