xref: /petsc/src/dm/impls/plex/plexcreate.c (revision 46139095832dbec5bd83c8636886c7e9bb646aaf)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h> /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>   /*I   "petscdmplex.h"   I*/
4 #include <petscsf.h>
5 #include <petscdmplextransform.h>
6 #include <petscdmlabelephemeral.h>
7 #include <petsc/private/kernels/blockmatmult.h>
8 #include <petsc/private/kernels/blockinvert.h>
9 
10 PetscLogEvent DMPLEX_CreateFromFile, DMPLEX_BuildFromCellList, DMPLEX_BuildCoordinatesFromCellList;
11 
12 /* External function declarations here */
13 static PetscErrorCode DMInitialize_Plex(DM dm);
14 
15 /* This copies internal things in the Plex structure that we generally want when making a new, related Plex */
16 PetscErrorCode DMPlexCopy_Internal(DM dmin, PetscBool copyPeriodicity, PetscBool copyOverlap, DM dmout)
17 {
18   const PetscReal         *maxCell, *Lstart, *L;
19   PetscBool                dist;
20   DMPlexReorderDefaultFlag reorder;
21 
22   PetscFunctionBegin;
23   if (copyPeriodicity) {
24     PetscCall(DMGetPeriodicity(dmin, &maxCell, &Lstart, &L));
25     PetscCall(DMSetPeriodicity(dmout, maxCell, Lstart, L));
26   }
27   PetscCall(DMPlexDistributeGetDefault(dmin, &dist));
28   PetscCall(DMPlexDistributeSetDefault(dmout, dist));
29   PetscCall(DMPlexReorderGetDefault(dmin, &reorder));
30   PetscCall(DMPlexReorderSetDefault(dmout, reorder));
31   ((DM_Plex *)dmout->data)->useHashLocation = ((DM_Plex *)dmin->data)->useHashLocation;
32   if (copyOverlap) PetscCall(DMPlexSetOverlap_Plex(dmout, dmin, 0));
33   PetscFunctionReturn(0);
34 }
35 
36 /* Replace dm with the contents of ndm, and then destroy ndm
37    - Share the DM_Plex structure
38    - Share the coordinates
39    - Share the SF
40 */
41 PetscErrorCode DMPlexReplace_Internal(DM dm, DM *ndm)
42 {
43   PetscSF          sf;
44   DM               dmNew = *ndm, coordDM, coarseDM;
45   Vec              coords;
46   const PetscReal *maxCell, *Lstart, *L;
47   PetscInt         dim, cdim;
48 
49   PetscFunctionBegin;
50   if (dm == dmNew) {
51     PetscCall(DMDestroy(ndm));
52     PetscFunctionReturn(0);
53   }
54   dm->setupcalled = dmNew->setupcalled;
55   PetscCall(DMGetDimension(dmNew, &dim));
56   PetscCall(DMSetDimension(dm, dim));
57   PetscCall(DMGetCoordinateDim(dmNew, &cdim));
58   PetscCall(DMSetCoordinateDim(dm, cdim));
59   PetscCall(DMGetPointSF(dmNew, &sf));
60   PetscCall(DMSetPointSF(dm, sf));
61   PetscCall(DMGetCoordinateDM(dmNew, &coordDM));
62   PetscCall(DMGetCoordinatesLocal(dmNew, &coords));
63   PetscCall(DMSetCoordinateDM(dm, coordDM));
64   PetscCall(DMSetCoordinatesLocal(dm, coords));
65   PetscCall(DMGetCellCoordinateDM(dmNew, &coordDM));
66   PetscCall(DMGetCellCoordinatesLocal(dmNew, &coords));
67   PetscCall(DMSetCellCoordinateDM(dm, coordDM));
68   PetscCall(DMSetCellCoordinatesLocal(dm, coords));
69   /* Do not want to create the coordinate field if it does not already exist, so do not call DMGetCoordinateField() */
70   PetscCall(DMFieldDestroy(&dm->coordinates[0].field));
71   dm->coordinates[0].field            = dmNew->coordinates[0].field;
72   ((DM_Plex *)dmNew->data)->coordFunc = ((DM_Plex *)dm->data)->coordFunc;
73   PetscCall(DMGetPeriodicity(dmNew, &maxCell, &Lstart, &L));
74   PetscCall(DMSetPeriodicity(dm, maxCell, Lstart, L));
75   PetscCall(DMDestroy_Plex(dm));
76   PetscCall(DMInitialize_Plex(dm));
77   dm->data = dmNew->data;
78   ((DM_Plex *)dmNew->data)->refct++;
79   PetscCall(DMPlexGetIsoperiodicFaceSF(dm, &sf));
80   PetscCall(DMPlexSetIsoperiodicFaceSF(dm, sf)); // for the compose function effect on dm
81   PetscCall(DMDestroyLabelLinkList_Internal(dm));
82   PetscCall(DMCopyLabels(dmNew, dm, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL));
83   PetscCall(DMGetCoarseDM(dmNew, &coarseDM));
84   PetscCall(DMSetCoarseDM(dm, coarseDM));
85   PetscCall(DMDestroy(ndm));
86   PetscFunctionReturn(0);
87 }
88 
89 /* Swap dm with the contents of dmNew
90    - Swap the DM_Plex structure
91    - Swap the coordinates
92    - Swap the point PetscSF
93 */
94 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
95 {
96   DM          coordDMA, coordDMB;
97   Vec         coordsA, coordsB;
98   PetscSF     sfA, sfB;
99   DMField     fieldTmp;
100   void       *tmp;
101   DMLabelLink listTmp;
102   DMLabel     depthTmp;
103   PetscInt    tmpI;
104 
105   PetscFunctionBegin;
106   if (dmA == dmB) PetscFunctionReturn(0);
107   PetscCall(DMGetPointSF(dmA, &sfA));
108   PetscCall(DMGetPointSF(dmB, &sfB));
109   PetscCall(PetscObjectReference((PetscObject)sfA));
110   PetscCall(DMSetPointSF(dmA, sfB));
111   PetscCall(DMSetPointSF(dmB, sfA));
112   PetscCall(PetscObjectDereference((PetscObject)sfA));
113 
114   PetscCall(DMGetCoordinateDM(dmA, &coordDMA));
115   PetscCall(DMGetCoordinateDM(dmB, &coordDMB));
116   PetscCall(PetscObjectReference((PetscObject)coordDMA));
117   PetscCall(DMSetCoordinateDM(dmA, coordDMB));
118   PetscCall(DMSetCoordinateDM(dmB, coordDMA));
119   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
120 
121   PetscCall(DMGetCoordinatesLocal(dmA, &coordsA));
122   PetscCall(DMGetCoordinatesLocal(dmB, &coordsB));
123   PetscCall(PetscObjectReference((PetscObject)coordsA));
124   PetscCall(DMSetCoordinatesLocal(dmA, coordsB));
125   PetscCall(DMSetCoordinatesLocal(dmB, coordsA));
126   PetscCall(PetscObjectDereference((PetscObject)coordsA));
127 
128   PetscCall(DMGetCellCoordinateDM(dmA, &coordDMA));
129   PetscCall(DMGetCellCoordinateDM(dmB, &coordDMB));
130   PetscCall(PetscObjectReference((PetscObject)coordDMA));
131   PetscCall(DMSetCellCoordinateDM(dmA, coordDMB));
132   PetscCall(DMSetCellCoordinateDM(dmB, coordDMA));
133   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
134 
135   PetscCall(DMGetCellCoordinatesLocal(dmA, &coordsA));
136   PetscCall(DMGetCellCoordinatesLocal(dmB, &coordsB));
137   PetscCall(PetscObjectReference((PetscObject)coordsA));
138   PetscCall(DMSetCellCoordinatesLocal(dmA, coordsB));
139   PetscCall(DMSetCellCoordinatesLocal(dmB, coordsA));
140   PetscCall(PetscObjectDereference((PetscObject)coordsA));
141 
142   fieldTmp                  = dmA->coordinates[0].field;
143   dmA->coordinates[0].field = dmB->coordinates[0].field;
144   dmB->coordinates[0].field = fieldTmp;
145   fieldTmp                  = dmA->coordinates[1].field;
146   dmA->coordinates[1].field = dmB->coordinates[1].field;
147   dmB->coordinates[1].field = fieldTmp;
148   tmp                       = dmA->data;
149   dmA->data                 = dmB->data;
150   dmB->data                 = tmp;
151   listTmp                   = dmA->labels;
152   dmA->labels               = dmB->labels;
153   dmB->labels               = listTmp;
154   depthTmp                  = dmA->depthLabel;
155   dmA->depthLabel           = dmB->depthLabel;
156   dmB->depthLabel           = depthTmp;
157   depthTmp                  = dmA->celltypeLabel;
158   dmA->celltypeLabel        = dmB->celltypeLabel;
159   dmB->celltypeLabel        = depthTmp;
160   tmpI                      = dmA->levelup;
161   dmA->levelup              = dmB->levelup;
162   dmB->levelup              = tmpI;
163   PetscFunctionReturn(0);
164 }
165 
166 PetscErrorCode DMPlexInterpolateInPlace_Internal(DM dm)
167 {
168   DM idm;
169 
170   PetscFunctionBegin;
171   PetscCall(DMPlexInterpolate(dm, &idm));
172   PetscCall(DMPlexCopyCoordinates(dm, idm));
173   PetscCall(DMPlexReplace_Internal(dm, &idm));
174   PetscFunctionReturn(0);
175 }
176 
177 /*@C
178   DMPlexCreateCoordinateSpace - Creates a finite element space for the coordinates
179 
180   Collective on dm
181 
182   Input Parameters:
183 + DM        - The DM
184 . degree    - The degree of the finite element or PETSC_DECIDE
185 - coordFunc - An optional function to map new points from refinement to the surface
186 
187   Level: advanced
188 
189 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `PetscPointFunc`, `PetscFECreateLagrange()`, `DMGetCoordinateDM()`
190 @*/
191 PetscErrorCode DMPlexCreateCoordinateSpace(DM dm, PetscInt degree, PetscPointFunc coordFunc)
192 {
193   DM_Plex     *mesh = (DM_Plex *)dm->data;
194   DM           cdm;
195   PetscDS      cds;
196   PetscFE      fe;
197   PetscClassId id;
198 
199   PetscFunctionBegin;
200   PetscCall(DMGetCoordinateDM(dm, &cdm));
201   PetscCall(DMGetDS(cdm, &cds));
202   PetscCall(PetscDSGetDiscretization(cds, 0, (PetscObject *)&fe));
203   PetscCall(PetscObjectGetClassId((PetscObject)fe, &id));
204   if (id != PETSCFE_CLASSID) {
205     PetscInt dim, dE, qorder;
206 
207     PetscCall(DMGetDimension(dm, &dim));
208     PetscCall(DMGetCoordinateDim(dm, &dE));
209     qorder = degree;
210     PetscObjectOptionsBegin((PetscObject)cdm);
211     PetscCall(PetscOptionsBoundedInt("-default_quadrature_order", "Quadrature order is one less than quadrature points per edge", "DMPlexCreateCoordinateSpace", qorder, &qorder, NULL, 0));
212     PetscOptionsEnd();
213     if (degree == PETSC_DECIDE) fe = NULL;
214     else {
215       DMPolytopeType ct;
216       PetscInt       cStart, cEnd, ctTmp;
217 
218       PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
219       // Want to match cell types
220       if (cEnd > cStart) PetscCall(DMPlexGetCellType(dm, cStart, &ct));
221       else ct = DM_POLYTOPE_UNKNOWN;
222       ctTmp = (PetscInt)ct;
223       PetscCallMPI(MPI_Allreduce(MPI_IN_PLACE, &ctTmp, 1, MPIU_INT, MPI_MIN, PetscObjectComm((PetscObject)dm)));
224       ct = (DMPolytopeType)ctTmp;
225       // Work around current bug
226       if (ct == DM_POLYTOPE_SEG_PRISM_TENSOR || ct == DM_POLYTOPE_TRI_PRISM_TENSOR || ct == DM_POLYTOPE_QUAD_PRISM_TENSOR) fe = NULL;
227       else PetscCall(PetscFECreateLagrangeByCell(PETSC_COMM_SELF, dim, dE, ct, degree, qorder, &fe));
228     }
229     if (fe) PetscCall(DMProjectCoordinates(dm, fe));
230     PetscCall(PetscFEDestroy(&fe));
231   }
232   mesh->coordFunc = coordFunc;
233   PetscFunctionReturn(0);
234 }
235 
236 /*@
237   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
238 
239   Collective
240 
241   Input Parameters:
242 + comm - The communicator for the `DM` object
243 . dim - The spatial dimension
244 . simplex - Flag for simplicial cells, otherwise they are tensor product cells
245 . interpolate - Flag to create intermediate mesh pieces (edges, faces)
246 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
247 
248   Output Parameter:
249 . dm - The `DM` object
250 
251   Level: beginner
252 
253 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMSetType()`, `DMCreate()`
254 @*/
255 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscReal refinementLimit, DM *newdm)
256 {
257   DM          dm;
258   PetscMPIInt rank;
259 
260   PetscFunctionBegin;
261   PetscCall(DMCreate(comm, &dm));
262   PetscCall(DMSetType(dm, DMPLEX));
263   PetscCall(DMSetDimension(dm, dim));
264   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
265   PetscCallMPI(MPI_Comm_rank(comm, &rank));
266   switch (dim) {
267   case 2:
268     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "triangular"));
269     else PetscCall(PetscObjectSetName((PetscObject)dm, "quadrilateral"));
270     break;
271   case 3:
272     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "tetrahedral"));
273     else PetscCall(PetscObjectSetName((PetscObject)dm, "hexahedral"));
274     break;
275   default:
276     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
277   }
278   if (rank) {
279     PetscInt numPoints[2] = {0, 0};
280     PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL));
281   } else {
282     switch (dim) {
283     case 2:
284       if (simplex) {
285         PetscInt    numPoints[2]        = {4, 2};
286         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
287         PetscInt    cones[6]            = {2, 3, 4, 5, 4, 3};
288         PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
289         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
290 
291         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
292       } else {
293         PetscInt    numPoints[2]        = {6, 2};
294         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
295         PetscInt    cones[8]            = {2, 3, 4, 5, 3, 6, 7, 4};
296         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
297         PetscScalar vertexCoords[12]    = {-1.0, -0.5, 0.0, -0.5, 0.0, 0.5, -1.0, 0.5, 1.0, -0.5, 1.0, 0.5};
298 
299         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
300       }
301       break;
302     case 3:
303       if (simplex) {
304         PetscInt    numPoints[2]        = {5, 2};
305         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
306         PetscInt    cones[8]            = {4, 3, 5, 2, 5, 3, 4, 6};
307         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
308         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0};
309 
310         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
311       } else {
312         PetscInt    numPoints[2]         = {12, 2};
313         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
314         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9, 5, 4, 10, 11, 7, 12, 13, 8};
315         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
316         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5, -1.0, 0.5, -0.5, 0.0, 0.5, -0.5, 0.0, -0.5, -0.5, -1.0, -0.5, 0.5, 0.0, -0.5, 0.5, 0.0, 0.5, 0.5, -1.0, 0.5, 0.5, 1.0, 0.5, -0.5, 1.0, -0.5, -0.5, 1.0, -0.5, 0.5, 1.0, 0.5, 0.5};
317 
318         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
319       }
320       break;
321     default:
322       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
323     }
324   }
325   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
326   *newdm = dm;
327   if (refinementLimit > 0.0) {
328     DM          rdm;
329     const char *name;
330 
331     PetscCall(DMPlexSetRefinementUniform(*newdm, PETSC_FALSE));
332     PetscCall(DMPlexSetRefinementLimit(*newdm, refinementLimit));
333     PetscCall(DMRefine(*newdm, comm, &rdm));
334     PetscCall(PetscObjectGetName((PetscObject)*newdm, &name));
335     PetscCall(PetscObjectSetName((PetscObject)rdm, name));
336     PetscCall(DMDestroy(newdm));
337     *newdm = rdm;
338   }
339   if (interpolate) {
340     DM idm;
341 
342     PetscCall(DMPlexInterpolate(*newdm, &idm));
343     PetscCall(DMDestroy(newdm));
344     *newdm = idm;
345   }
346   PetscFunctionReturn(0);
347 }
348 
349 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
350 {
351   const PetscInt numVertices    = 2;
352   PetscInt       markerRight    = 1;
353   PetscInt       markerLeft     = 1;
354   PetscBool      markerSeparate = PETSC_FALSE;
355   Vec            coordinates;
356   PetscSection   coordSection;
357   PetscScalar   *coords;
358   PetscInt       coordSize;
359   PetscMPIInt    rank;
360   PetscInt       cdim = 1, v;
361 
362   PetscFunctionBegin;
363   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
364   if (markerSeparate) {
365     markerRight = 2;
366     markerLeft  = 1;
367   }
368   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
369   if (rank == 0) {
370     PetscCall(DMPlexSetChart(dm, 0, numVertices));
371     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
372     PetscCall(DMSetLabelValue(dm, "marker", 0, markerLeft));
373     PetscCall(DMSetLabelValue(dm, "marker", 1, markerRight));
374   }
375   PetscCall(DMPlexSymmetrize(dm));
376   PetscCall(DMPlexStratify(dm));
377   /* Build coordinates */
378   PetscCall(DMSetCoordinateDim(dm, cdim));
379   PetscCall(DMGetCoordinateSection(dm, &coordSection));
380   PetscCall(PetscSectionSetNumFields(coordSection, 1));
381   PetscCall(PetscSectionSetChart(coordSection, 0, numVertices));
382   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
383   for (v = 0; v < numVertices; ++v) {
384     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
385     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
386   }
387   PetscCall(PetscSectionSetUp(coordSection));
388   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
389   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
390   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
391   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
392   PetscCall(VecSetBlockSize(coordinates, cdim));
393   PetscCall(VecSetType(coordinates, VECSTANDARD));
394   PetscCall(VecGetArray(coordinates, &coords));
395   coords[0] = lower[0];
396   coords[1] = upper[0];
397   PetscCall(VecRestoreArray(coordinates, &coords));
398   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
399   PetscCall(VecDestroy(&coordinates));
400   PetscFunctionReturn(0);
401 }
402 
403 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
404 {
405   const PetscInt numVertices    = (edges[0] + 1) * (edges[1] + 1);
406   const PetscInt numEdges       = edges[0] * (edges[1] + 1) + (edges[0] + 1) * edges[1];
407   PetscInt       markerTop      = 1;
408   PetscInt       markerBottom   = 1;
409   PetscInt       markerRight    = 1;
410   PetscInt       markerLeft     = 1;
411   PetscBool      markerSeparate = PETSC_FALSE;
412   Vec            coordinates;
413   PetscSection   coordSection;
414   PetscScalar   *coords;
415   PetscInt       coordSize;
416   PetscMPIInt    rank;
417   PetscInt       v, vx, vy;
418 
419   PetscFunctionBegin;
420   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
421   if (markerSeparate) {
422     markerTop    = 3;
423     markerBottom = 1;
424     markerRight  = 2;
425     markerLeft   = 4;
426   }
427   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
428   if (rank == 0) {
429     PetscInt e, ex, ey;
430 
431     PetscCall(DMPlexSetChart(dm, 0, numEdges + numVertices));
432     for (e = 0; e < numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
433     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
434     for (vx = 0; vx <= edges[0]; vx++) {
435       for (ey = 0; ey < edges[1]; ey++) {
436         PetscInt edge   = vx * edges[1] + ey + edges[0] * (edges[1] + 1);
437         PetscInt vertex = ey * (edges[0] + 1) + vx + numEdges;
438         PetscInt cone[2];
439 
440         cone[0] = vertex;
441         cone[1] = vertex + edges[0] + 1;
442         PetscCall(DMPlexSetCone(dm, edge, cone));
443         if (vx == edges[0]) {
444           PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
445           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
446           if (ey == edges[1] - 1) {
447             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
448             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerRight));
449           }
450         } else if (vx == 0) {
451           PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
452           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
453           if (ey == edges[1] - 1) {
454             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
455             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft));
456           }
457         }
458       }
459     }
460     for (vy = 0; vy <= edges[1]; vy++) {
461       for (ex = 0; ex < edges[0]; ex++) {
462         PetscInt edge   = vy * edges[0] + ex;
463         PetscInt vertex = vy * (edges[0] + 1) + ex + numEdges;
464         PetscInt cone[2];
465 
466         cone[0] = vertex;
467         cone[1] = vertex + 1;
468         PetscCall(DMPlexSetCone(dm, edge, cone));
469         if (vy == edges[1]) {
470           PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
471           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
472           if (ex == edges[0] - 1) {
473             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
474             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerTop));
475           }
476         } else if (vy == 0) {
477           PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
478           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
479           if (ex == edges[0] - 1) {
480             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
481             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom));
482           }
483         }
484       }
485     }
486   }
487   PetscCall(DMPlexSymmetrize(dm));
488   PetscCall(DMPlexStratify(dm));
489   /* Build coordinates */
490   PetscCall(DMSetCoordinateDim(dm, 2));
491   PetscCall(DMGetCoordinateSection(dm, &coordSection));
492   PetscCall(PetscSectionSetNumFields(coordSection, 1));
493   PetscCall(PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices));
494   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 2));
495   for (v = numEdges; v < numEdges + numVertices; ++v) {
496     PetscCall(PetscSectionSetDof(coordSection, v, 2));
497     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 2));
498   }
499   PetscCall(PetscSectionSetUp(coordSection));
500   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
501   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
502   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
503   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
504   PetscCall(VecSetBlockSize(coordinates, 2));
505   PetscCall(VecSetType(coordinates, VECSTANDARD));
506   PetscCall(VecGetArray(coordinates, &coords));
507   for (vy = 0; vy <= edges[1]; ++vy) {
508     for (vx = 0; vx <= edges[0]; ++vx) {
509       coords[(vy * (edges[0] + 1) + vx) * 2 + 0] = lower[0] + ((upper[0] - lower[0]) / edges[0]) * vx;
510       coords[(vy * (edges[0] + 1) + vx) * 2 + 1] = lower[1] + ((upper[1] - lower[1]) / edges[1]) * vy;
511     }
512   }
513   PetscCall(VecRestoreArray(coordinates, &coords));
514   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
515   PetscCall(VecDestroy(&coordinates));
516   PetscFunctionReturn(0);
517 }
518 
519 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
520 {
521   PetscInt     vertices[3], numVertices;
522   PetscInt     numFaces       = 2 * faces[0] * faces[1] + 2 * faces[1] * faces[2] + 2 * faces[0] * faces[2];
523   PetscInt     markerTop      = 1;
524   PetscInt     markerBottom   = 1;
525   PetscInt     markerFront    = 1;
526   PetscInt     markerBack     = 1;
527   PetscInt     markerRight    = 1;
528   PetscInt     markerLeft     = 1;
529   PetscBool    markerSeparate = PETSC_FALSE;
530   Vec          coordinates;
531   PetscSection coordSection;
532   PetscScalar *coords;
533   PetscInt     coordSize;
534   PetscMPIInt  rank;
535   PetscInt     v, vx, vy, vz;
536   PetscInt     voffset, iface = 0, cone[4];
537 
538   PetscFunctionBegin;
539   PetscCheck(faces[0] >= 1 && faces[1] >= 1 && faces[2] >= 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
540   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
541   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
542   if (markerSeparate) {
543     markerBottom = 1;
544     markerTop    = 2;
545     markerFront  = 3;
546     markerBack   = 4;
547     markerRight  = 5;
548     markerLeft   = 6;
549   }
550   vertices[0] = faces[0] + 1;
551   vertices[1] = faces[1] + 1;
552   vertices[2] = faces[2] + 1;
553   numVertices = vertices[0] * vertices[1] * vertices[2];
554   if (rank == 0) {
555     PetscInt f;
556 
557     PetscCall(DMPlexSetChart(dm, 0, numFaces + numVertices));
558     for (f = 0; f < numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
559     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
560 
561     /* Side 0 (Top) */
562     for (vy = 0; vy < faces[1]; vy++) {
563       for (vx = 0; vx < faces[0]; vx++) {
564         voffset = numFaces + vertices[0] * vertices[1] * (vertices[2] - 1) + vy * vertices[0] + vx;
565         cone[0] = voffset;
566         cone[1] = voffset + 1;
567         cone[2] = voffset + vertices[0] + 1;
568         cone[3] = voffset + vertices[0];
569         PetscCall(DMPlexSetCone(dm, iface, cone));
570         PetscCall(DMSetLabelValue(dm, "marker", iface, markerTop));
571         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerTop));
572         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerTop));
573         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerTop));
574         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerTop));
575         iface++;
576       }
577     }
578 
579     /* Side 1 (Bottom) */
580     for (vy = 0; vy < faces[1]; vy++) {
581       for (vx = 0; vx < faces[0]; vx++) {
582         voffset = numFaces + vy * (faces[0] + 1) + vx;
583         cone[0] = voffset + 1;
584         cone[1] = voffset;
585         cone[2] = voffset + vertices[0];
586         cone[3] = voffset + vertices[0] + 1;
587         PetscCall(DMPlexSetCone(dm, iface, cone));
588         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBottom));
589         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBottom));
590         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBottom));
591         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerBottom));
592         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerBottom));
593         iface++;
594       }
595     }
596 
597     /* Side 2 (Front) */
598     for (vz = 0; vz < faces[2]; vz++) {
599       for (vx = 0; vx < faces[0]; vx++) {
600         voffset = numFaces + vz * vertices[0] * vertices[1] + vx;
601         cone[0] = voffset;
602         cone[1] = voffset + 1;
603         cone[2] = voffset + vertices[0] * vertices[1] + 1;
604         cone[3] = voffset + vertices[0] * vertices[1];
605         PetscCall(DMPlexSetCone(dm, iface, cone));
606         PetscCall(DMSetLabelValue(dm, "marker", iface, markerFront));
607         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerFront));
608         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerFront));
609         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerFront));
610         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerFront));
611         iface++;
612       }
613     }
614 
615     /* Side 3 (Back) */
616     for (vz = 0; vz < faces[2]; vz++) {
617       for (vx = 0; vx < faces[0]; vx++) {
618         voffset = numFaces + vz * vertices[0] * vertices[1] + vertices[0] * (vertices[1] - 1) + vx;
619         cone[0] = voffset + vertices[0] * vertices[1];
620         cone[1] = voffset + vertices[0] * vertices[1] + 1;
621         cone[2] = voffset + 1;
622         cone[3] = voffset;
623         PetscCall(DMPlexSetCone(dm, iface, cone));
624         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBack));
625         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBack));
626         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBack));
627         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerBack));
628         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerBack));
629         iface++;
630       }
631     }
632 
633     /* Side 4 (Left) */
634     for (vz = 0; vz < faces[2]; vz++) {
635       for (vy = 0; vy < faces[1]; vy++) {
636         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0];
637         cone[0] = voffset;
638         cone[1] = voffset + vertices[0] * vertices[1];
639         cone[2] = voffset + vertices[0] * vertices[1] + vertices[0];
640         cone[3] = voffset + vertices[0];
641         PetscCall(DMPlexSetCone(dm, iface, cone));
642         PetscCall(DMSetLabelValue(dm, "marker", iface, markerLeft));
643         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerLeft));
644         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerLeft));
645         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[1] + 0, markerLeft));
646         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerLeft));
647         iface++;
648       }
649     }
650 
651     /* Side 5 (Right) */
652     for (vz = 0; vz < faces[2]; vz++) {
653       for (vy = 0; vy < faces[1]; vy++) {
654         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0] + faces[0];
655         cone[0] = voffset + vertices[0] * vertices[1];
656         cone[1] = voffset;
657         cone[2] = voffset + vertices[0];
658         cone[3] = voffset + vertices[0] * vertices[1] + vertices[0];
659         PetscCall(DMPlexSetCone(dm, iface, cone));
660         PetscCall(DMSetLabelValue(dm, "marker", iface, markerRight));
661         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerRight));
662         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerRight));
663         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerRight));
664         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerRight));
665         iface++;
666       }
667     }
668   }
669   PetscCall(DMPlexSymmetrize(dm));
670   PetscCall(DMPlexStratify(dm));
671   /* Build coordinates */
672   PetscCall(DMSetCoordinateDim(dm, 3));
673   PetscCall(DMGetCoordinateSection(dm, &coordSection));
674   PetscCall(PetscSectionSetNumFields(coordSection, 1));
675   PetscCall(PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices));
676   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 3));
677   for (v = numFaces; v < numFaces + numVertices; ++v) {
678     PetscCall(PetscSectionSetDof(coordSection, v, 3));
679     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 3));
680   }
681   PetscCall(PetscSectionSetUp(coordSection));
682   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
683   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
684   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
685   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
686   PetscCall(VecSetBlockSize(coordinates, 3));
687   PetscCall(VecSetType(coordinates, VECSTANDARD));
688   PetscCall(VecGetArray(coordinates, &coords));
689   for (vz = 0; vz <= faces[2]; ++vz) {
690     for (vy = 0; vy <= faces[1]; ++vy) {
691       for (vx = 0; vx <= faces[0]; ++vx) {
692         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 0] = lower[0] + ((upper[0] - lower[0]) / faces[0]) * vx;
693         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 1] = lower[1] + ((upper[1] - lower[1]) / faces[1]) * vy;
694         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 2] = lower[2] + ((upper[2] - lower[2]) / faces[2]) * vz;
695       }
696     }
697   }
698   PetscCall(VecRestoreArray(coordinates, &coords));
699   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
700   PetscCall(VecDestroy(&coordinates));
701   PetscFunctionReturn(0);
702 }
703 
704 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate)
705 {
706   PetscFunctionBegin;
707   PetscValidLogicalCollectiveInt(dm, dim, 2);
708   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
709   PetscCall(DMSetDimension(dm, dim - 1));
710   PetscCall(DMSetCoordinateDim(dm, dim));
711   switch (dim) {
712   case 1:
713     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(dm, lower, upper, faces));
714     break;
715   case 2:
716     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(dm, lower, upper, faces));
717     break;
718   case 3:
719     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(dm, lower, upper, faces));
720     break;
721   default:
722     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Dimension not supported: %" PetscInt_FMT, dim);
723   }
724   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
725   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
726   PetscFunctionReturn(0);
727 }
728 
729 /*@C
730   DMPlexCreateBoxSurfaceMesh - Creates a mesh on the surface of the tensor product of unit intervals (box) using tensor cells (hexahedra).
731 
732   Collective
733 
734   Input Parameters:
735 + comm        - The communicator for the `DM` object
736 . dim         - The spatial dimension of the box, so the resulting mesh is has dimension dim-1
737 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
738 . lower       - The lower left corner, or NULL for (0, 0, 0)
739 . upper       - The upper right corner, or NULL for (1, 1, 1)
740 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
741 
742   Output Parameter:
743 . dm  - The `DM` object
744 
745   Level: beginner
746 
747 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateBoxMesh()`, `DMPlexCreateFromFile()`, `DMSetType()`, `DMCreate()`
748 @*/
749 PetscErrorCode DMPlexCreateBoxSurfaceMesh(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate, DM *dm)
750 {
751   PetscInt  fac[3] = {1, 1, 1};
752   PetscReal low[3] = {0, 0, 0};
753   PetscReal upp[3] = {1, 1, 1};
754 
755   PetscFunctionBegin;
756   PetscCall(DMCreate(comm, dm));
757   PetscCall(DMSetType(*dm, DMPLEX));
758   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(*dm, dim, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, interpolate));
759   PetscFunctionReturn(0);
760 }
761 
762 static PetscErrorCode DMPlexCreateLineMesh_Internal(DM dm, PetscInt segments, PetscReal lower, PetscReal upper, DMBoundaryType bd)
763 {
764   PetscInt     i, fStart, fEnd, numCells = 0, numVerts = 0;
765   PetscInt     numPoints[2], *coneSize, *cones, *coneOrientations;
766   PetscScalar *vertexCoords;
767   PetscReal    L, maxCell;
768   PetscBool    markerSeparate = PETSC_FALSE;
769   PetscInt     markerLeft = 1, faceMarkerLeft = 1;
770   PetscInt     markerRight = 1, faceMarkerRight = 2;
771   PetscBool    wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
772   PetscMPIInt  rank;
773 
774   PetscFunctionBegin;
775   PetscValidPointer(dm, 1);
776 
777   PetscCall(DMSetDimension(dm, 1));
778   PetscCall(DMCreateLabel(dm, "marker"));
779   PetscCall(DMCreateLabel(dm, "Face Sets"));
780 
781   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
782   if (rank == 0) numCells = segments;
783   if (rank == 0) numVerts = segments + (wrap ? 0 : 1);
784 
785   numPoints[0] = numVerts;
786   numPoints[1] = numCells;
787   PetscCall(PetscMalloc4(numCells + numVerts, &coneSize, numCells * 2, &cones, numCells + numVerts, &coneOrientations, numVerts, &vertexCoords));
788   PetscCall(PetscArrayzero(coneOrientations, numCells + numVerts));
789   for (i = 0; i < numCells; ++i) coneSize[i] = 2;
790   for (i = 0; i < numVerts; ++i) coneSize[numCells + i] = 0;
791   for (i = 0; i < numCells; ++i) {
792     cones[2 * i]     = numCells + i % numVerts;
793     cones[2 * i + 1] = numCells + (i + 1) % numVerts;
794   }
795   for (i = 0; i < numVerts; ++i) vertexCoords[i] = lower + (upper - lower) * ((PetscReal)i / (PetscReal)numCells);
796   PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
797   PetscCall(PetscFree4(coneSize, cones, coneOrientations, vertexCoords));
798 
799   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
800   if (markerSeparate) {
801     markerLeft  = faceMarkerLeft;
802     markerRight = faceMarkerRight;
803   }
804   if (!wrap && rank == 0) {
805     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
806     PetscCall(DMSetLabelValue(dm, "marker", fStart, markerLeft));
807     PetscCall(DMSetLabelValue(dm, "marker", fEnd - 1, markerRight));
808     PetscCall(DMSetLabelValue(dm, "Face Sets", fStart, faceMarkerLeft));
809     PetscCall(DMSetLabelValue(dm, "Face Sets", fEnd - 1, faceMarkerRight));
810   }
811   if (wrap) {
812     L       = upper - lower;
813     maxCell = (PetscReal)1.1 * (L / (PetscReal)PetscMax(1, segments));
814     PetscCall(DMSetPeriodicity(dm, &maxCell, &lower, &L));
815   }
816   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
817   PetscFunctionReturn(0);
818 }
819 
820 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
821 {
822   DM      boundary, vol;
823   DMLabel bdlabel;
824 
825   PetscFunctionBegin;
826   PetscValidPointer(dm, 1);
827   for (PetscInt i = 0; i < dim; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
828   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &boundary));
829   PetscCall(DMSetType(boundary, DMPLEX));
830   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(boundary, dim, faces, lower, upper, PETSC_FALSE));
831   PetscCall(DMPlexGenerate(boundary, NULL, interpolate, &vol));
832   PetscCall(DMGetLabel(vol, "marker", &bdlabel));
833   if (bdlabel) PetscCall(DMPlexLabelComplete(vol, bdlabel));
834   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_FALSE, vol));
835   PetscCall(DMPlexReplace_Internal(dm, &vol));
836   PetscCall(DMDestroy(&boundary));
837   PetscFunctionReturn(0);
838 }
839 
840 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
841 {
842   DMLabel     cutLabel  = NULL;
843   PetscInt    markerTop = 1, faceMarkerTop = 1;
844   PetscInt    markerBottom = 1, faceMarkerBottom = 1;
845   PetscInt    markerFront = 1, faceMarkerFront = 1;
846   PetscInt    markerBack = 1, faceMarkerBack = 1;
847   PetscInt    markerRight = 1, faceMarkerRight = 1;
848   PetscInt    markerLeft = 1, faceMarkerLeft = 1;
849   PetscInt    dim;
850   PetscBool   markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
851   PetscMPIInt rank;
852 
853   PetscFunctionBegin;
854   PetscCall(DMGetDimension(dm, &dim));
855   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
856   PetscCall(DMCreateLabel(dm, "marker"));
857   PetscCall(DMCreateLabel(dm, "Face Sets"));
858   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
859   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST || bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST || bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
860     if (cutMarker) {
861       PetscCall(DMCreateLabel(dm, "periodic_cut"));
862       PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
863     }
864   }
865   switch (dim) {
866   case 2:
867     faceMarkerTop    = 3;
868     faceMarkerBottom = 1;
869     faceMarkerRight  = 2;
870     faceMarkerLeft   = 4;
871     break;
872   case 3:
873     faceMarkerBottom = 1;
874     faceMarkerTop    = 2;
875     faceMarkerFront  = 3;
876     faceMarkerBack   = 4;
877     faceMarkerRight  = 5;
878     faceMarkerLeft   = 6;
879     break;
880   default:
881     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
882   }
883   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
884   if (markerSeparate) {
885     markerBottom = faceMarkerBottom;
886     markerTop    = faceMarkerTop;
887     markerFront  = faceMarkerFront;
888     markerBack   = faceMarkerBack;
889     markerRight  = faceMarkerRight;
890     markerLeft   = faceMarkerLeft;
891   }
892   {
893     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
894     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
895     const PetscInt numZEdges    = rank == 0 ? edges[2] : 0;
896     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0] + 1) : 0;
897     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1] + 1) : 0;
898     const PetscInt numZVertices = rank == 0 ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2] + 1) : 0;
899     const PetscInt numCells     = numXEdges * numYEdges * numZEdges;
900     const PetscInt numXFaces    = numYEdges * numZEdges;
901     const PetscInt numYFaces    = numXEdges * numZEdges;
902     const PetscInt numZFaces    = numXEdges * numYEdges;
903     const PetscInt numTotXFaces = numXVertices * numXFaces;
904     const PetscInt numTotYFaces = numYVertices * numYFaces;
905     const PetscInt numTotZFaces = numZVertices * numZFaces;
906     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
907     const PetscInt numTotXEdges = numXEdges * numYVertices * numZVertices;
908     const PetscInt numTotYEdges = numYEdges * numXVertices * numZVertices;
909     const PetscInt numTotZEdges = numZEdges * numXVertices * numYVertices;
910     const PetscInt numVertices  = numXVertices * numYVertices * numZVertices;
911     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
912     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
913     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
914     const PetscInt firstYFace   = firstXFace + numTotXFaces;
915     const PetscInt firstZFace   = firstYFace + numTotYFaces;
916     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
917     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
918     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
919     Vec            coordinates;
920     PetscSection   coordSection;
921     PetscScalar   *coords;
922     PetscInt       coordSize;
923     PetscInt       v, vx, vy, vz;
924     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
925 
926     PetscCall(DMPlexSetChart(dm, 0, numCells + numFaces + numEdges + numVertices));
927     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
928     for (f = firstXFace; f < firstXFace + numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
929     for (e = firstXEdge; e < firstXEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
930     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
931     /* Build cells */
932     for (fz = 0; fz < numZEdges; ++fz) {
933       for (fy = 0; fy < numYEdges; ++fy) {
934         for (fx = 0; fx < numXEdges; ++fx) {
935           PetscInt cell  = (fz * numYEdges + fy) * numXEdges + fx;
936           PetscInt faceB = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
937           PetscInt faceT = firstZFace + (fy * numXEdges + fx) * numZVertices + ((fz + 1) % numZVertices);
938           PetscInt faceF = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
939           PetscInt faceK = firstYFace + (fz * numXEdges + fx) * numYVertices + ((fy + 1) % numYVertices);
940           PetscInt faceL = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
941           PetscInt faceR = firstXFace + (fz * numYEdges + fy) * numXVertices + ((fx + 1) % numXVertices);
942           /* B,  T,  F,  K,  R,  L */
943           PetscInt ornt[6] = {-2, 0, 0, -3, 0, -2}; /* ??? */
944           PetscInt cone[6];
945 
946           /* no boundary twisting in 3D */
947           cone[0] = faceB;
948           cone[1] = faceT;
949           cone[2] = faceF;
950           cone[3] = faceK;
951           cone[4] = faceR;
952           cone[5] = faceL;
953           PetscCall(DMPlexSetCone(dm, cell, cone));
954           PetscCall(DMPlexSetConeOrientation(dm, cell, ornt));
955           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
956           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
957           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
958         }
959       }
960     }
961     /* Build x faces */
962     for (fz = 0; fz < numZEdges; ++fz) {
963       for (fy = 0; fy < numYEdges; ++fy) {
964         for (fx = 0; fx < numXVertices; ++fx) {
965           PetscInt face    = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
966           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
967           PetscInt edgeR   = firstZEdge + (((fy + 1) % numYVertices) * numXVertices + fx) * numZEdges + fz;
968           PetscInt edgeB   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
969           PetscInt edgeT   = firstYEdge + (((fz + 1) % numZVertices) * numXVertices + fx) * numYEdges + fy;
970           PetscInt ornt[4] = {0, 0, -1, -1};
971           PetscInt cone[4];
972 
973           if (dim == 3) {
974             /* markers */
975             if (bdX != DM_BOUNDARY_PERIODIC) {
976               if (fx == numXVertices - 1) {
977                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight));
978                 PetscCall(DMSetLabelValue(dm, "marker", face, markerRight));
979               } else if (fx == 0) {
980                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft));
981                 PetscCall(DMSetLabelValue(dm, "marker", face, markerLeft));
982               }
983             }
984           }
985           cone[0] = edgeB;
986           cone[1] = edgeR;
987           cone[2] = edgeT;
988           cone[3] = edgeL;
989           PetscCall(DMPlexSetCone(dm, face, cone));
990           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
991         }
992       }
993     }
994     /* Build y faces */
995     for (fz = 0; fz < numZEdges; ++fz) {
996       for (fx = 0; fx < numXEdges; ++fx) {
997         for (fy = 0; fy < numYVertices; ++fy) {
998           PetscInt face    = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
999           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
1000           PetscInt edgeR   = firstZEdge + (fy * numXVertices + ((fx + 1) % numXVertices)) * numZEdges + fz;
1001           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1002           PetscInt edgeT   = firstXEdge + (((fz + 1) % numZVertices) * numYVertices + fy) * numXEdges + fx;
1003           PetscInt ornt[4] = {0, 0, -1, -1};
1004           PetscInt cone[4];
1005 
1006           if (dim == 3) {
1007             /* markers */
1008             if (bdY != DM_BOUNDARY_PERIODIC) {
1009               if (fy == numYVertices - 1) {
1010                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack));
1011                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBack));
1012               } else if (fy == 0) {
1013                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront));
1014                 PetscCall(DMSetLabelValue(dm, "marker", face, markerFront));
1015               }
1016             }
1017           }
1018           cone[0] = edgeB;
1019           cone[1] = edgeR;
1020           cone[2] = edgeT;
1021           cone[3] = edgeL;
1022           PetscCall(DMPlexSetCone(dm, face, cone));
1023           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1024         }
1025       }
1026     }
1027     /* Build z faces */
1028     for (fy = 0; fy < numYEdges; ++fy) {
1029       for (fx = 0; fx < numXEdges; ++fx) {
1030         for (fz = 0; fz < numZVertices; fz++) {
1031           PetscInt face    = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
1032           PetscInt edgeL   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
1033           PetscInt edgeR   = firstYEdge + (fz * numXVertices + ((fx + 1) % numXVertices)) * numYEdges + fy;
1034           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1035           PetscInt edgeT   = firstXEdge + (fz * numYVertices + ((fy + 1) % numYVertices)) * numXEdges + fx;
1036           PetscInt ornt[4] = {0, 0, -1, -1};
1037           PetscInt cone[4];
1038 
1039           if (dim == 2) {
1040             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges - 1) {
1041               edgeR += numYEdges - 1 - 2 * fy;
1042               ornt[1] = -1;
1043             }
1044             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges - 1) {
1045               edgeT += numXEdges - 1 - 2 * fx;
1046               ornt[2] = 0;
1047             }
1048             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1049             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1050           } else {
1051             /* markers */
1052             if (bdZ != DM_BOUNDARY_PERIODIC) {
1053               if (fz == numZVertices - 1) {
1054                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop));
1055                 PetscCall(DMSetLabelValue(dm, "marker", face, markerTop));
1056               } else if (fz == 0) {
1057                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom));
1058                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBottom));
1059               }
1060             }
1061           }
1062           cone[0] = edgeB;
1063           cone[1] = edgeR;
1064           cone[2] = edgeT;
1065           cone[3] = edgeL;
1066           PetscCall(DMPlexSetCone(dm, face, cone));
1067           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1068         }
1069       }
1070     }
1071     /* Build Z edges*/
1072     for (vy = 0; vy < numYVertices; vy++) {
1073       for (vx = 0; vx < numXVertices; vx++) {
1074         for (ez = 0; ez < numZEdges; ez++) {
1075           const PetscInt edge    = firstZEdge + (vy * numXVertices + vx) * numZEdges + ez;
1076           const PetscInt vertexB = firstVertex + (ez * numYVertices + vy) * numXVertices + vx;
1077           const PetscInt vertexT = firstVertex + (((ez + 1) % numZVertices) * numYVertices + vy) * numXVertices + vx;
1078           PetscInt       cone[2];
1079 
1080           cone[0] = vertexB;
1081           cone[1] = vertexT;
1082           PetscCall(DMPlexSetCone(dm, edge, cone));
1083           if (dim == 3) {
1084             if (bdX != DM_BOUNDARY_PERIODIC) {
1085               if (vx == numXVertices - 1) {
1086                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1087                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1088                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1089               } else if (vx == 0) {
1090                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1091                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1092                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1093               }
1094             }
1095             if (bdY != DM_BOUNDARY_PERIODIC) {
1096               if (vy == numYVertices - 1) {
1097                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1098                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1099                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1100               } else if (vy == 0) {
1101                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1102                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1103                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1104               }
1105             }
1106           }
1107         }
1108       }
1109     }
1110     /* Build Y edges*/
1111     for (vz = 0; vz < numZVertices; vz++) {
1112       for (vx = 0; vx < numXVertices; vx++) {
1113         for (ey = 0; ey < numYEdges; ey++) {
1114           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges - 1) ? (numXVertices - vx - 1) : (vz * numYVertices + ((ey + 1) % numYVertices)) * numXVertices + vx;
1115           const PetscInt edge    = firstYEdge + (vz * numXVertices + vx) * numYEdges + ey;
1116           const PetscInt vertexF = firstVertex + (vz * numYVertices + ey) * numXVertices + vx;
1117           const PetscInt vertexK = firstVertex + nextv;
1118           PetscInt       cone[2];
1119 
1120           cone[0] = vertexF;
1121           cone[1] = vertexK;
1122           PetscCall(DMPlexSetCone(dm, edge, cone));
1123           if (dim == 2) {
1124             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1125               if (vx == numXVertices - 1) {
1126                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight));
1127                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1128                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1129                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1130               } else if (vx == 0) {
1131                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft));
1132                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1133                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1134                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1135               }
1136             } else {
1137               if (vx == 0 && cutLabel) {
1138                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1139                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1140                 if (ey == numYEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1141               }
1142             }
1143           } else {
1144             if (bdX != DM_BOUNDARY_PERIODIC) {
1145               if (vx == numXVertices - 1) {
1146                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1147                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1148                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1149               } else if (vx == 0) {
1150                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1151                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1152                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1153               }
1154             }
1155             if (bdZ != DM_BOUNDARY_PERIODIC) {
1156               if (vz == numZVertices - 1) {
1157                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1158                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1159                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1160               } else if (vz == 0) {
1161                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1162                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1163                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1164               }
1165             }
1166           }
1167         }
1168       }
1169     }
1170     /* Build X edges*/
1171     for (vz = 0; vz < numZVertices; vz++) {
1172       for (vy = 0; vy < numYVertices; vy++) {
1173         for (ex = 0; ex < numXEdges; ex++) {
1174           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges - 1) ? (numYVertices - vy - 1) * numXVertices : (vz * numYVertices + vy) * numXVertices + (ex + 1) % numXVertices;
1175           const PetscInt edge    = firstXEdge + (vz * numYVertices + vy) * numXEdges + ex;
1176           const PetscInt vertexL = firstVertex + (vz * numYVertices + vy) * numXVertices + ex;
1177           const PetscInt vertexR = firstVertex + nextv;
1178           PetscInt       cone[2];
1179 
1180           cone[0] = vertexL;
1181           cone[1] = vertexR;
1182           PetscCall(DMPlexSetCone(dm, edge, cone));
1183           if (dim == 2) {
1184             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1185               if (vy == numYVertices - 1) {
1186                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop));
1187                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1188                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1189                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1190               } else if (vy == 0) {
1191                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom));
1192                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1193                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1194                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1195               }
1196             } else {
1197               if (vy == 0 && cutLabel) {
1198                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1199                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1200                 if (ex == numXEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1201               }
1202             }
1203           } else {
1204             if (bdY != DM_BOUNDARY_PERIODIC) {
1205               if (vy == numYVertices - 1) {
1206                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1207                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1208                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1209               } else if (vy == 0) {
1210                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1211                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1212                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1213               }
1214             }
1215             if (bdZ != DM_BOUNDARY_PERIODIC) {
1216               if (vz == numZVertices - 1) {
1217                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1218                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1219                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1220               } else if (vz == 0) {
1221                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1222                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1223                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1224               }
1225             }
1226           }
1227         }
1228       }
1229     }
1230     PetscCall(DMPlexSymmetrize(dm));
1231     PetscCall(DMPlexStratify(dm));
1232     /* Build coordinates */
1233     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1234     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1235     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1236     PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVertices));
1237     for (v = firstVertex; v < firstVertex + numVertices; ++v) {
1238       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1239       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1240     }
1241     PetscCall(PetscSectionSetUp(coordSection));
1242     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1243     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1244     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1245     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1246     PetscCall(VecSetBlockSize(coordinates, dim));
1247     PetscCall(VecSetType(coordinates, VECSTANDARD));
1248     PetscCall(VecGetArray(coordinates, &coords));
1249     for (vz = 0; vz < numZVertices; ++vz) {
1250       for (vy = 0; vy < numYVertices; ++vy) {
1251         for (vx = 0; vx < numXVertices; ++vx) {
1252           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * vx;
1253           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * vy;
1254           if (dim == 3) coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 2] = lower[2] + ((upper[2] - lower[2]) / numZEdges) * vz;
1255         }
1256       }
1257     }
1258     PetscCall(VecRestoreArray(coordinates, &coords));
1259     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1260     PetscCall(VecDestroy(&coordinates));
1261   }
1262   PetscFunctionReturn(0);
1263 }
1264 
1265 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1266 {
1267   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1268   PetscInt       fac[3] = {0, 0, 0}, d;
1269 
1270   PetscFunctionBegin;
1271   PetscValidPointer(dm, 1);
1272   PetscValidLogicalCollectiveInt(dm, dim, 2);
1273   PetscCall(DMSetDimension(dm, dim));
1274   for (d = 0; d < dim; ++d) {
1275     fac[d] = faces[d];
1276     bdt[d] = periodicity[d];
1277   }
1278   PetscCall(DMPlexCreateCubeMesh_Internal(dm, lower, upper, fac, bdt[0], bdt[1], bdt[2]));
1279   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST || periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST || (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
1280     PetscReal L[3]       = {-1., -1., 0.};
1281     PetscReal maxCell[3] = {-1., -1., 0.};
1282 
1283     for (d = 0; d < dim; ++d) {
1284       if (periodicity[d] != DM_BOUNDARY_NONE) {
1285         L[d]       = upper[d] - lower[d];
1286         maxCell[d] = 1.1 * (L[d] / PetscMax(1, faces[d]));
1287       }
1288     }
1289     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1290   }
1291   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
1292   PetscFunctionReturn(0);
1293 }
1294 
1295 static PetscErrorCode DMPlexCreateBoxMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1296 {
1297   PetscBool box_sfc = PETSC_FALSE;
1298   PetscFunctionBegin;
1299   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
1300   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_box_sfc", &box_sfc, NULL));
1301   if (box_sfc) PetscCall(DMPlexCreateBoxMesh_Tensor_SFC_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1302   else if (dim == 1) PetscCall(DMPlexCreateLineMesh_Internal(dm, faces[0], lower[0], upper[0], periodicity[0]));
1303   else if (simplex) PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1304   else PetscCall(DMPlexCreateBoxMesh_Tensor_Internal(dm, dim, faces, lower, upper, periodicity));
1305   if (!interpolate && dim > 1 && !simplex) {
1306     DM udm;
1307 
1308     PetscCall(DMPlexUninterpolate(dm, &udm));
1309     PetscCall(DMPlexCopyCoordinates(dm, udm));
1310     PetscCall(DMPlexReplace_Internal(dm, &udm));
1311   }
1312   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
1313   PetscFunctionReturn(0);
1314 }
1315 
1316 /*@C
1317   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
1318 
1319   Collective
1320 
1321   Input Parameters:
1322 + comm        - The communicator for the `DM` object
1323 . dim         - The spatial dimension
1324 . simplex     - `PETSC_TRUE` for simplices, `PETSC_FALSE` for tensor cells
1325 . faces       - Number of faces per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1326 . lower       - The lower left corner, or NULL for (0, 0, 0)
1327 . upper       - The upper right corner, or NULL for (1, 1, 1)
1328 . periodicity - The boundary type for the X,Y,Z direction, or NULL for `DM_BOUNDARY_NONE`
1329 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1330 
1331   Output Parameter:
1332 . dm  - The `DM` object
1333 
1334   Level: beginner
1335 
1336   Note:
1337    To customize this mesh using options, use
1338 .vb
1339   DMCreate(comm, &dm);
1340   DMSetType(dm, DMPLEX);
1341   DMSetFromOptions(dm);
1342 .ve
1343 and use the options in `DMSetFromOptions()`.
1344 
1345   Here is the numbering returned for 2 faces in each direction for tensor cells:
1346 .vb
1347  10---17---11---18----12
1348   |         |         |
1349   |         |         |
1350  20    2   22    3    24
1351   |         |         |
1352   |         |         |
1353   7---15----8---16----9
1354   |         |         |
1355   |         |         |
1356  19    0   21    1   23
1357   |         |         |
1358   |         |         |
1359   4---13----5---14----6
1360 .ve
1361 and for simplicial cells
1362 .vb
1363  14----8---15----9----16
1364   |\     5  |\      7 |
1365   | \       | \       |
1366  13   2    14    3    15
1367   | 4   \   | 6   \   |
1368   |       \ |       \ |
1369  11----6---12----7----13
1370   |\        |\        |
1371   | \    1  | \     3 |
1372  10   0    11    1    12
1373   | 0   \   | 2   \   |
1374   |       \ |       \ |
1375   8----4----9----5----10
1376 .ve
1377 
1378 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
1379 @*/
1380 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
1381 {
1382   PetscInt       fac[3] = {1, 1, 1};
1383   PetscReal      low[3] = {0, 0, 0};
1384   PetscReal      upp[3] = {1, 1, 1};
1385   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1386 
1387   PetscFunctionBegin;
1388   PetscCall(DMCreate(comm, dm));
1389   PetscCall(DMSetType(*dm, DMPLEX));
1390   PetscCall(DMPlexCreateBoxMesh_Internal(*dm, dim, simplex, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt, interpolate));
1391   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1392   PetscFunctionReturn(0);
1393 }
1394 
1395 static PetscErrorCode DMPlexCreateWedgeBoxMesh_Internal(DM dm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1396 {
1397   DM       bdm, vol;
1398   PetscInt i;
1399 
1400   PetscFunctionBegin;
1401   for (i = 0; i < 3; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity not yet supported");
1402   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &bdm));
1403   PetscCall(DMSetType(bdm, DMPLEX));
1404   PetscCall(DMSetDimension(bdm, 2));
1405   PetscCall(PetscLogEventBegin(DMPLEX_Generate, bdm, 0, 0, 0));
1406   PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(bdm, 2, faces, lower, upper, periodicity, PETSC_TRUE));
1407   PetscCall(DMPlexExtrude(bdm, faces[2], upper[2] - lower[2], PETSC_TRUE, PETSC_FALSE, NULL, NULL, &vol));
1408   PetscCall(PetscLogEventEnd(DMPLEX_Generate, bdm, 0, 0, 0));
1409   PetscCall(DMDestroy(&bdm));
1410   PetscCall(DMPlexReplace_Internal(dm, &vol));
1411   if (lower[2] != 0.0) {
1412     Vec          v;
1413     PetscScalar *x;
1414     PetscInt     cDim, n;
1415 
1416     PetscCall(DMGetCoordinatesLocal(dm, &v));
1417     PetscCall(VecGetBlockSize(v, &cDim));
1418     PetscCall(VecGetLocalSize(v, &n));
1419     PetscCall(VecGetArray(v, &x));
1420     x += cDim;
1421     for (i = 0; i < n; i += cDim) x[i] += lower[2];
1422     PetscCall(VecRestoreArray(v, &x));
1423     PetscCall(DMSetCoordinatesLocal(dm, v));
1424   }
1425   PetscFunctionReturn(0);
1426 }
1427 
1428 /*@
1429   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tesselating the (x,y) plane and extruding in the third direction using wedge cells.
1430 
1431   Collective
1432 
1433   Input Parameters:
1434 + comm        - The communicator for the `DM` object
1435 . faces       - Number of faces per dimension, or NULL for (1, 1, 1)
1436 . lower       - The lower left corner, or NULL for (0, 0, 0)
1437 . upper       - The upper right corner, or NULL for (1, 1, 1)
1438 . periodicity - The boundary type for the X,Y,Z direction, or NULL for `DM_BOUNDARY_NONE`
1439 . orderHeight - If `PETSC_TRUE`, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1440 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1441 
1442   Output Parameter:
1443 . dm  - The `DM` object
1444 
1445   Level: beginner
1446 
1447 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateWedgeCylinderMesh()`, `DMExtrude()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1448 @*/
1449 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool orderHeight, PetscBool interpolate, DM *dm)
1450 {
1451   PetscInt       fac[3] = {1, 1, 1};
1452   PetscReal      low[3] = {0, 0, 0};
1453   PetscReal      upp[3] = {1, 1, 1};
1454   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1455 
1456   PetscFunctionBegin;
1457   PetscCall(DMCreate(comm, dm));
1458   PetscCall(DMSetType(*dm, DMPLEX));
1459   PetscCall(DMPlexCreateWedgeBoxMesh_Internal(*dm, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt));
1460   if (!interpolate) {
1461     DM udm;
1462 
1463     PetscCall(DMPlexUninterpolate(*dm, &udm));
1464     PetscCall(DMPlexReplace_Internal(*dm, &udm));
1465   }
1466   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1467   PetscFunctionReturn(0);
1468 }
1469 
1470 /*
1471   DMPlexTensorPointLexicographic_Private - Returns all tuples of size 'len' with nonnegative integers that are all less than or equal to 'max' for that dimension.
1472                                            Ordering is lexicographic with lowest index as least significant in ordering.
1473                                            e.g. for len == 2 and max == 2, this will return, in order, {0,0}, {1,0}, {2,0}, {0,1}, {1,1}, {2,1}, {0,2}, {1,2}, {2,2}.
1474 
1475   Input Parameters:
1476 + len - The length of the tuple
1477 . max - The maximum for each dimension, so values are in [0, max)
1478 - tup - A tuple of length len+1: tup[len] > 0 indicates a stopping condition
1479 
1480   Output Parameter:
1481 . tup - A tuple of len integers whose entries are at most 'max'
1482 
1483   Level: developer
1484 
1485 .seealso: PetscDualSpaceTensorPointLexicographic_Internal(), PetscDualSpaceLatticePointLexicographic_Internal()
1486 */
1487 static PetscErrorCode DMPlexTensorPointLexicographic_Private(PetscInt len, const PetscInt max[], PetscInt tup[])
1488 {
1489   PetscInt i;
1490 
1491   PetscFunctionBegin;
1492   for (i = 0; i < len; ++i) {
1493     if (tup[i] < max[i] - 1) {
1494       break;
1495     } else {
1496       tup[i] = 0;
1497     }
1498   }
1499   if (i == len) tup[i - 1] = max[i - 1];
1500   else ++tup[i];
1501   PetscFunctionReturn(0);
1502 }
1503 
1504 static PetscInt TupleToIndex_Private(PetscInt len, const PetscInt max[], const PetscInt tup[])
1505 {
1506   PetscInt i, idx = tup[len - 1];
1507 
1508   for (i = len - 2; i >= 0; --i) {
1509     idx *= max[i];
1510     idx += tup[i];
1511   }
1512   return idx;
1513 }
1514 
1515 static PetscErrorCode DestroyExtent_Private(void *extent)
1516 {
1517   return PetscFree(extent);
1518 }
1519 
1520 static PetscErrorCode DMPlexCreateHypercubicMesh_Internal(DM dm, PetscInt dim, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], const DMBoundaryType bd[])
1521 {
1522   Vec          coordinates;
1523   PetscSection coordSection;
1524   DMLabel      cutLabel    = NULL;
1525   PetscBool    cutMarker   = PETSC_FALSE;
1526   PetscBool    periodic    = PETSC_FALSE;
1527   PetscInt     numCells    = 1, c;
1528   PetscInt     numVertices = 1, v;
1529   PetscScalar *coords;
1530   PetscInt    *vertices, *vert, *vtmp, *supp, cone[2];
1531   PetscInt     d, e, cell = 0, coordSize;
1532   PetscMPIInt  rank;
1533 
1534   PetscFunctionBegin;
1535   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1536   PetscCall(DMSetDimension(dm, dim));
1537   PetscCall(PetscCalloc4(dim, &vertices, dim, &vert, dim, &vtmp, 2 * dim, &supp));
1538   PetscCall(DMCreateLabel(dm, "marker"));
1539   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
1540   for (d = 0; d < dim; ++d) periodic = (periodic || bd[d] == DM_BOUNDARY_PERIODIC) ? PETSC_TRUE : PETSC_FALSE;
1541   if (periodic && cutMarker) {
1542     PetscCall(DMCreateLabel(dm, "periodic_cut"));
1543     PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
1544   }
1545   for (d = 0; d < dim; ++d) PetscCheck(bd[d] == DM_BOUNDARY_PERIODIC, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Hypercubic mesh must be periodic now");
1546   for (d = 0; d < dim; ++d) {
1547     vertices[d] = edges[d];
1548     numVertices *= vertices[d];
1549   }
1550   numCells = numVertices * dim;
1551   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1552   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, 2));
1553   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetSupportSize(dm, v, 2 * dim));
1554   /* TODO Loop over boundary and reset support sizes */
1555   PetscCall(DMSetUp(dm)); /* Allocate space for cones and supports */
1556   /* Build cell cones and vertex supports */
1557   PetscCall(DMCreateLabel(dm, "celltype"));
1558   while (vert[dim - 1] < vertices[dim - 1]) {
1559     const PetscInt vertex = TupleToIndex_Private(dim, vertices, vert) + numCells;
1560     PetscInt       s      = 0;
1561 
1562     PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT ":", vertex);
1563     for (d = 0; d < dim; ++d) PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, vert[d]);
1564     PetscPrintf(PETSC_COMM_SELF, "\n");
1565     PetscCall(DMPlexSetCellType(dm, vertex, DM_POLYTOPE_POINT));
1566     for (d = 0; d < dim; ++d) {
1567       for (e = 0; e < dim; ++e) vtmp[e] = vert[e];
1568       vtmp[d] = (vert[d] + 1) % vertices[d];
1569       cone[0] = vertex;
1570       cone[1] = TupleToIndex_Private(dim, vertices, vtmp) + numCells;
1571       PetscPrintf(PETSC_COMM_SELF, "  Vertex %" PetscInt_FMT ":", cone[1]);
1572       for (e = 0; e < dim; ++e) PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, vtmp[e]);
1573       PetscPrintf(PETSC_COMM_SELF, "\n");
1574       PetscCall(DMPlexSetCone(dm, cell, cone));
1575       PetscCall(DMPlexSetCellType(dm, cell, DM_POLYTOPE_SEGMENT));
1576       PetscPrintf(PETSC_COMM_SELF, "  Edge %" PetscInt_FMT " (%" PetscInt_FMT " %" PetscInt_FMT ")\n", cell, cone[0], cone[1]);
1577       ++cell;
1578     }
1579     for (d = 0; d < dim; ++d) {
1580       for (e = 0; e < dim; ++e) vtmp[e] = vert[e];
1581       vtmp[d]   = (vert[d] + vertices[d] - 1) % vertices[d];
1582       supp[s++] = TupleToIndex_Private(dim, vertices, vtmp) * dim + d;
1583       supp[s++] = (vertex - numCells) * dim + d;
1584       PetscCall(DMPlexSetSupport(dm, vertex, supp));
1585     }
1586     PetscCall(DMPlexTensorPointLexicographic_Private(dim, vertices, vert));
1587   }
1588   PetscCall(DMPlexStratify(dm));
1589   /* Build coordinates */
1590   PetscCall(DMGetCoordinateSection(dm, &coordSection));
1591   PetscCall(PetscSectionSetNumFields(coordSection, 1));
1592   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1593   PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1594   for (v = numCells; v < numCells + numVertices; ++v) {
1595     PetscCall(PetscSectionSetDof(coordSection, v, dim));
1596     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1597   }
1598   PetscCall(PetscSectionSetUp(coordSection));
1599   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1600   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1601   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1602   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1603   PetscCall(VecSetBlockSize(coordinates, dim));
1604   PetscCall(VecSetType(coordinates, VECSTANDARD));
1605   PetscCall(VecGetArray(coordinates, &coords));
1606   for (d = 0; d < dim; ++d) vert[d] = 0;
1607   while (vert[dim - 1] < vertices[dim - 1]) {
1608     const PetscInt vertex = TupleToIndex_Private(dim, vertices, vert);
1609 
1610     for (d = 0; d < dim; ++d) coords[vertex * dim + d] = lower[d] + ((upper[d] - lower[d]) / vertices[d]) * vert[d];
1611     PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT ":", vertex);
1612     for (d = 0; d < dim; ++d) PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, vert[d]);
1613     for (d = 0; d < dim; ++d) PetscPrintf(PETSC_COMM_SELF, " %g", (double)PetscRealPart(coords[vertex * dim + d]));
1614     PetscPrintf(PETSC_COMM_SELF, "\n");
1615     PetscCall(DMPlexTensorPointLexicographic_Private(dim, vertices, vert));
1616   }
1617   PetscCall(VecRestoreArray(coordinates, &coords));
1618   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1619   PetscCall(VecDestroy(&coordinates));
1620   PetscCall(PetscFree4(vertices, vert, vtmp, supp));
1621   //PetscCall(DMSetPeriodicity(dm, NULL, lower, upper));
1622   // Attach the extent
1623   {
1624     PetscContainer c;
1625     PetscInt      *extent;
1626 
1627     PetscCall(PetscMalloc1(dim, &extent));
1628     for (PetscInt d = 0; d < dim; ++d) extent[d] = edges[d];
1629     PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &c));
1630     PetscCall(PetscContainerSetUserDestroy(c, DestroyExtent_Private));
1631     PetscCall(PetscContainerSetPointer(c, extent));
1632     PetscCall(PetscObjectCompose((PetscObject)dm, "_extent", (PetscObject)c));
1633     PetscCall(PetscContainerDestroy(&c));
1634   }
1635   PetscFunctionReturn(0);
1636 }
1637 
1638 /*@C
1639   DMPlexCreateHypercubicMesh - Creates a peridoic mesh on the tensor product of unit intervals using only vertices and edges.
1640 
1641   Collective
1642 
1643   Input Parameters:
1644 + comm        - The communicator for the DM object
1645 . dim         - The spatial dimension
1646 . edges       - Number of edges per dimension, or NULL for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1647 . lower       - The lower left corner, or NULL for (0, 0, 0)
1648 - upper       - The upper right corner, or NULL for (1, 1, 1)
1649 
1650   Output Parameter:
1651 . dm  - The DM object
1652 
1653   Note: If you want to customize this mesh using options, you just need to
1654 $  DMCreate(comm, &dm);
1655 $  DMSetType(dm, DMPLEX);
1656 $  DMSetFromOptions(dm);
1657 and use the options on the DMSetFromOptions() page.
1658 
1659   The vertices are numbered is lexicographic order, and the dim edges exiting a vertex in the positive orthant are number consecutively,
1660 $ 18--0-19--2-20--4-18
1661 $  |     |     |     |
1662 $ 13    15    17    13
1663 $  |     |     |     |
1664 $ 24-12-25-14-26-16-24
1665 $  |     |     |     |
1666 $  7     9    11     7
1667 $  |     |     |     |
1668 $ 21--6-22--8-23-10-21
1669 $  |     |     |     |
1670 $  1     3     5     1
1671 $  |     |     |     |
1672 $ 18--0-19--2-20--4-18
1673 
1674   Level: beginner
1675 
1676 .seealso: DMSetFromOptions(), DMPlexCreateFromFile(), DMPlexCreateHexCylinderMesh(), DMSetType(), DMCreate()
1677 @*/
1678 PetscErrorCode DMPlexCreateHypercubicMesh(MPI_Comm comm, PetscInt dim, const PetscInt edges[], const PetscReal lower[], const PetscReal upper[], DM *dm)
1679 {
1680   PetscInt       *edg;
1681   PetscReal      *low, *upp;
1682   DMBoundaryType *bdt;
1683   PetscInt        d;
1684 
1685   PetscFunctionBegin;
1686   PetscCall(DMCreate(comm, dm));
1687   PetscCall(DMSetType(*dm, DMPLEX));
1688   PetscCall(PetscMalloc4(dim, &edg, dim, &low, dim, &upp, dim, &bdt));
1689   for (d = 0; d < dim; ++d) {
1690     edg[d] = edges ? edges[d] : 1;
1691     low[d] = lower ? lower[d] : 0.;
1692     upp[d] = upper ? upper[d] : 1.;
1693     bdt[d] = DM_BOUNDARY_PERIODIC;
1694   }
1695   PetscCall(DMPlexCreateHypercubicMesh_Internal(*dm, dim, low, upp, edg, bdt));
1696   PetscCall(PetscFree4(edg, low, upp, bdt));
1697   PetscFunctionReturn(0);
1698 }
1699 
1700 /*@C
1701   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all `DM` options in the database.
1702 
1703   Logically Collective on dm
1704 
1705   Input Parameters:
1706 + dm - the DM context
1707 - prefix - the prefix to prepend to all option names
1708 
1709   Level: advanced
1710 
1711   Note:
1712   A hyphen (-) must NOT be given at the beginning of the prefix name.
1713   The first character of all runtime options is AUTOMATICALLY the hyphen.
1714 
1715 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `SNESSetFromOptions()`
1716 @*/
1717 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
1718 {
1719   DM_Plex *mesh = (DM_Plex *)dm->data;
1720 
1721   PetscFunctionBegin;
1722   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1723   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, prefix));
1724   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)mesh->partitioner, prefix));
1725   PetscFunctionReturn(0);
1726 }
1727 
1728 /* Remap geometry to cylinder
1729    TODO: This only works for a single refinement, then it is broken
1730 
1731      Interior square: Linear interpolation is correct
1732      The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
1733      such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
1734 
1735        phi     = arctan(y/x)
1736        d_close = sqrt(1/8 + 1/4 sin^2(phi))
1737        d_far   = sqrt(1/2 + sin^2(phi))
1738 
1739      so we remap them using
1740 
1741        x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
1742        y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
1743 
1744      If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
1745 */
1746 static void snapToCylinder(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
1747 {
1748   const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1749   const PetscReal ds2 = 0.5 * dis;
1750 
1751   if ((PetscAbsScalar(u[0]) <= ds2) && (PetscAbsScalar(u[1]) <= ds2)) {
1752     f0[0] = u[0];
1753     f0[1] = u[1];
1754   } else {
1755     PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
1756 
1757     x    = PetscRealPart(u[0]);
1758     y    = PetscRealPart(u[1]);
1759     phi  = PetscAtan2Real(y, x);
1760     sinp = PetscSinReal(phi);
1761     cosp = PetscCosReal(phi);
1762     if ((PetscAbsReal(phi) > PETSC_PI / 4.0) && (PetscAbsReal(phi) < 3.0 * PETSC_PI / 4.0)) {
1763       dc = PetscAbsReal(ds2 / sinp);
1764       df = PetscAbsReal(dis / sinp);
1765       xc = ds2 * x / PetscAbsReal(y);
1766       yc = ds2 * PetscSignReal(y);
1767     } else {
1768       dc = PetscAbsReal(ds2 / cosp);
1769       df = PetscAbsReal(dis / cosp);
1770       xc = ds2 * PetscSignReal(x);
1771       yc = ds2 * y / PetscAbsReal(x);
1772     }
1773     f0[0] = xc + (u[0] - xc) * (1.0 - dc) / (df - dc);
1774     f0[1] = yc + (u[1] - yc) * (1.0 - dc) / (df - dc);
1775   }
1776   f0[2] = u[2];
1777 }
1778 
1779 static PetscErrorCode DMPlexCreateHexCylinderMesh_Internal(DM dm, DMBoundaryType periodicZ)
1780 {
1781   const PetscInt dim = 3;
1782   PetscInt       numCells, numVertices;
1783   PetscMPIInt    rank;
1784 
1785   PetscFunctionBegin;
1786   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
1787   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1788   PetscCall(DMSetDimension(dm, dim));
1789   /* Create topology */
1790   {
1791     PetscInt cone[8], c;
1792 
1793     numCells    = rank == 0 ? 5 : 0;
1794     numVertices = rank == 0 ? 16 : 0;
1795     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1796       numCells *= 3;
1797       numVertices = rank == 0 ? 24 : 0;
1798     }
1799     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1800     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 8));
1801     PetscCall(DMSetUp(dm));
1802     if (rank == 0) {
1803       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1804         cone[0] = 15;
1805         cone[1] = 18;
1806         cone[2] = 17;
1807         cone[3] = 16;
1808         cone[4] = 31;
1809         cone[5] = 32;
1810         cone[6] = 33;
1811         cone[7] = 34;
1812         PetscCall(DMPlexSetCone(dm, 0, cone));
1813         cone[0] = 16;
1814         cone[1] = 17;
1815         cone[2] = 24;
1816         cone[3] = 23;
1817         cone[4] = 32;
1818         cone[5] = 36;
1819         cone[6] = 37;
1820         cone[7] = 33; /* 22 25 26 21 */
1821         PetscCall(DMPlexSetCone(dm, 1, cone));
1822         cone[0] = 18;
1823         cone[1] = 27;
1824         cone[2] = 24;
1825         cone[3] = 17;
1826         cone[4] = 34;
1827         cone[5] = 33;
1828         cone[6] = 37;
1829         cone[7] = 38;
1830         PetscCall(DMPlexSetCone(dm, 2, cone));
1831         cone[0] = 29;
1832         cone[1] = 27;
1833         cone[2] = 18;
1834         cone[3] = 15;
1835         cone[4] = 35;
1836         cone[5] = 31;
1837         cone[6] = 34;
1838         cone[7] = 38;
1839         PetscCall(DMPlexSetCone(dm, 3, cone));
1840         cone[0] = 29;
1841         cone[1] = 15;
1842         cone[2] = 16;
1843         cone[3] = 23;
1844         cone[4] = 35;
1845         cone[5] = 36;
1846         cone[6] = 32;
1847         cone[7] = 31;
1848         PetscCall(DMPlexSetCone(dm, 4, cone));
1849 
1850         cone[0] = 31;
1851         cone[1] = 34;
1852         cone[2] = 33;
1853         cone[3] = 32;
1854         cone[4] = 19;
1855         cone[5] = 22;
1856         cone[6] = 21;
1857         cone[7] = 20;
1858         PetscCall(DMPlexSetCone(dm, 5, cone));
1859         cone[0] = 32;
1860         cone[1] = 33;
1861         cone[2] = 37;
1862         cone[3] = 36;
1863         cone[4] = 22;
1864         cone[5] = 25;
1865         cone[6] = 26;
1866         cone[7] = 21;
1867         PetscCall(DMPlexSetCone(dm, 6, cone));
1868         cone[0] = 34;
1869         cone[1] = 38;
1870         cone[2] = 37;
1871         cone[3] = 33;
1872         cone[4] = 20;
1873         cone[5] = 21;
1874         cone[6] = 26;
1875         cone[7] = 28;
1876         PetscCall(DMPlexSetCone(dm, 7, cone));
1877         cone[0] = 35;
1878         cone[1] = 38;
1879         cone[2] = 34;
1880         cone[3] = 31;
1881         cone[4] = 30;
1882         cone[5] = 19;
1883         cone[6] = 20;
1884         cone[7] = 28;
1885         PetscCall(DMPlexSetCone(dm, 8, cone));
1886         cone[0] = 35;
1887         cone[1] = 31;
1888         cone[2] = 32;
1889         cone[3] = 36;
1890         cone[4] = 30;
1891         cone[5] = 25;
1892         cone[6] = 22;
1893         cone[7] = 19;
1894         PetscCall(DMPlexSetCone(dm, 9, cone));
1895 
1896         cone[0] = 19;
1897         cone[1] = 20;
1898         cone[2] = 21;
1899         cone[3] = 22;
1900         cone[4] = 15;
1901         cone[5] = 16;
1902         cone[6] = 17;
1903         cone[7] = 18;
1904         PetscCall(DMPlexSetCone(dm, 10, cone));
1905         cone[0] = 22;
1906         cone[1] = 21;
1907         cone[2] = 26;
1908         cone[3] = 25;
1909         cone[4] = 16;
1910         cone[5] = 23;
1911         cone[6] = 24;
1912         cone[7] = 17;
1913         PetscCall(DMPlexSetCone(dm, 11, cone));
1914         cone[0] = 20;
1915         cone[1] = 28;
1916         cone[2] = 26;
1917         cone[3] = 21;
1918         cone[4] = 18;
1919         cone[5] = 17;
1920         cone[6] = 24;
1921         cone[7] = 27;
1922         PetscCall(DMPlexSetCone(dm, 12, cone));
1923         cone[0] = 30;
1924         cone[1] = 28;
1925         cone[2] = 20;
1926         cone[3] = 19;
1927         cone[4] = 29;
1928         cone[5] = 15;
1929         cone[6] = 18;
1930         cone[7] = 27;
1931         PetscCall(DMPlexSetCone(dm, 13, cone));
1932         cone[0] = 30;
1933         cone[1] = 19;
1934         cone[2] = 22;
1935         cone[3] = 25;
1936         cone[4] = 29;
1937         cone[5] = 23;
1938         cone[6] = 16;
1939         cone[7] = 15;
1940         PetscCall(DMPlexSetCone(dm, 14, cone));
1941       } else {
1942         cone[0] = 5;
1943         cone[1] = 8;
1944         cone[2] = 7;
1945         cone[3] = 6;
1946         cone[4] = 9;
1947         cone[5] = 12;
1948         cone[6] = 11;
1949         cone[7] = 10;
1950         PetscCall(DMPlexSetCone(dm, 0, cone));
1951         cone[0] = 6;
1952         cone[1] = 7;
1953         cone[2] = 14;
1954         cone[3] = 13;
1955         cone[4] = 12;
1956         cone[5] = 15;
1957         cone[6] = 16;
1958         cone[7] = 11;
1959         PetscCall(DMPlexSetCone(dm, 1, cone));
1960         cone[0] = 8;
1961         cone[1] = 17;
1962         cone[2] = 14;
1963         cone[3] = 7;
1964         cone[4] = 10;
1965         cone[5] = 11;
1966         cone[6] = 16;
1967         cone[7] = 18;
1968         PetscCall(DMPlexSetCone(dm, 2, cone));
1969         cone[0] = 19;
1970         cone[1] = 17;
1971         cone[2] = 8;
1972         cone[3] = 5;
1973         cone[4] = 20;
1974         cone[5] = 9;
1975         cone[6] = 10;
1976         cone[7] = 18;
1977         PetscCall(DMPlexSetCone(dm, 3, cone));
1978         cone[0] = 19;
1979         cone[1] = 5;
1980         cone[2] = 6;
1981         cone[3] = 13;
1982         cone[4] = 20;
1983         cone[5] = 15;
1984         cone[6] = 12;
1985         cone[7] = 9;
1986         PetscCall(DMPlexSetCone(dm, 4, cone));
1987       }
1988     }
1989     PetscCall(DMPlexSymmetrize(dm));
1990     PetscCall(DMPlexStratify(dm));
1991   }
1992   /* Create cube geometry */
1993   {
1994     Vec             coordinates;
1995     PetscSection    coordSection;
1996     PetscScalar    *coords;
1997     PetscInt        coordSize, v;
1998     const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1999     const PetscReal ds2 = dis / 2.0;
2000 
2001     /* Build coordinates */
2002     PetscCall(DMGetCoordinateSection(dm, &coordSection));
2003     PetscCall(PetscSectionSetNumFields(coordSection, 1));
2004     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
2005     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
2006     for (v = numCells; v < numCells + numVertices; ++v) {
2007       PetscCall(PetscSectionSetDof(coordSection, v, dim));
2008       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
2009     }
2010     PetscCall(PetscSectionSetUp(coordSection));
2011     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2012     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2013     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2014     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2015     PetscCall(VecSetBlockSize(coordinates, dim));
2016     PetscCall(VecSetType(coordinates, VECSTANDARD));
2017     PetscCall(VecGetArray(coordinates, &coords));
2018     if (rank == 0) {
2019       coords[0 * dim + 0]  = -ds2;
2020       coords[0 * dim + 1]  = -ds2;
2021       coords[0 * dim + 2]  = 0.0;
2022       coords[1 * dim + 0]  = ds2;
2023       coords[1 * dim + 1]  = -ds2;
2024       coords[1 * dim + 2]  = 0.0;
2025       coords[2 * dim + 0]  = ds2;
2026       coords[2 * dim + 1]  = ds2;
2027       coords[2 * dim + 2]  = 0.0;
2028       coords[3 * dim + 0]  = -ds2;
2029       coords[3 * dim + 1]  = ds2;
2030       coords[3 * dim + 2]  = 0.0;
2031       coords[4 * dim + 0]  = -ds2;
2032       coords[4 * dim + 1]  = -ds2;
2033       coords[4 * dim + 2]  = 1.0;
2034       coords[5 * dim + 0]  = -ds2;
2035       coords[5 * dim + 1]  = ds2;
2036       coords[5 * dim + 2]  = 1.0;
2037       coords[6 * dim + 0]  = ds2;
2038       coords[6 * dim + 1]  = ds2;
2039       coords[6 * dim + 2]  = 1.0;
2040       coords[7 * dim + 0]  = ds2;
2041       coords[7 * dim + 1]  = -ds2;
2042       coords[7 * dim + 2]  = 1.0;
2043       coords[8 * dim + 0]  = dis;
2044       coords[8 * dim + 1]  = -dis;
2045       coords[8 * dim + 2]  = 0.0;
2046       coords[9 * dim + 0]  = dis;
2047       coords[9 * dim + 1]  = dis;
2048       coords[9 * dim + 2]  = 0.0;
2049       coords[10 * dim + 0] = dis;
2050       coords[10 * dim + 1] = -dis;
2051       coords[10 * dim + 2] = 1.0;
2052       coords[11 * dim + 0] = dis;
2053       coords[11 * dim + 1] = dis;
2054       coords[11 * dim + 2] = 1.0;
2055       coords[12 * dim + 0] = -dis;
2056       coords[12 * dim + 1] = dis;
2057       coords[12 * dim + 2] = 0.0;
2058       coords[13 * dim + 0] = -dis;
2059       coords[13 * dim + 1] = dis;
2060       coords[13 * dim + 2] = 1.0;
2061       coords[14 * dim + 0] = -dis;
2062       coords[14 * dim + 1] = -dis;
2063       coords[14 * dim + 2] = 0.0;
2064       coords[15 * dim + 0] = -dis;
2065       coords[15 * dim + 1] = -dis;
2066       coords[15 * dim + 2] = 1.0;
2067       if (periodicZ == DM_BOUNDARY_PERIODIC) {
2068         /* 15 31 19 */ coords[16 * dim + 0] = -ds2;
2069         coords[16 * dim + 1]                = -ds2;
2070         coords[16 * dim + 2]                = 0.5;
2071         /* 16 32 22 */ coords[17 * dim + 0] = ds2;
2072         coords[17 * dim + 1]                = -ds2;
2073         coords[17 * dim + 2]                = 0.5;
2074         /* 17 33 21 */ coords[18 * dim + 0] = ds2;
2075         coords[18 * dim + 1]                = ds2;
2076         coords[18 * dim + 2]                = 0.5;
2077         /* 18 34 20 */ coords[19 * dim + 0] = -ds2;
2078         coords[19 * dim + 1]                = ds2;
2079         coords[19 * dim + 2]                = 0.5;
2080         /* 29 35 30 */ coords[20 * dim + 0] = -dis;
2081         coords[20 * dim + 1]                = -dis;
2082         coords[20 * dim + 2]                = 0.5;
2083         /* 23 36 25 */ coords[21 * dim + 0] = dis;
2084         coords[21 * dim + 1]                = -dis;
2085         coords[21 * dim + 2]                = 0.5;
2086         /* 24 37 26 */ coords[22 * dim + 0] = dis;
2087         coords[22 * dim + 1]                = dis;
2088         coords[22 * dim + 2]                = 0.5;
2089         /* 27 38 28 */ coords[23 * dim + 0] = -dis;
2090         coords[23 * dim + 1]                = dis;
2091         coords[23 * dim + 2]                = 0.5;
2092       }
2093     }
2094     PetscCall(VecRestoreArray(coordinates, &coords));
2095     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2096     PetscCall(VecDestroy(&coordinates));
2097   }
2098   /* Create periodicity */
2099   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
2100     PetscReal L[3]       = {-1., -1., 0.};
2101     PetscReal maxCell[3] = {-1., -1., 0.};
2102     PetscReal lower[3]   = {0.0, 0.0, 0.0};
2103     PetscReal upper[3]   = {1.0, 1.0, 1.5};
2104     PetscInt  numZCells  = 3;
2105 
2106     L[2]       = upper[2] - lower[2];
2107     maxCell[2] = 1.1 * (L[2] / numZCells);
2108     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
2109   }
2110   {
2111     DM          cdm;
2112     PetscDS     cds;
2113     PetscScalar c[2] = {1.0, 1.0};
2114 
2115     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToCylinder));
2116     PetscCall(DMGetCoordinateDM(dm, &cdm));
2117     PetscCall(DMGetDS(cdm, &cds));
2118     PetscCall(PetscDSSetConstants(cds, 2, c));
2119   }
2120   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
2121 
2122   /* Wait for coordinate creation before doing in-place modification */
2123   PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2124   PetscFunctionReturn(0);
2125 }
2126 
2127 /*@
2128   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
2129 
2130   Collective
2131 
2132   Input Parameters:
2133 + comm      - The communicator for the `DM` object
2134 - periodicZ - The boundary type for the Z direction
2135 
2136   Output Parameter:
2137 . dm  - The DM object
2138 
2139   Level: beginner
2140 
2141   Note:
2142   Here is the output numbering looking from the bottom of the cylinder:
2143 .vb
2144        17-----14
2145         |     |
2146         |  2  |
2147         |     |
2148  17-----8-----7-----14
2149   |     |     |     |
2150   |  3  |  0  |  1  |
2151   |     |     |     |
2152  19-----5-----6-----13
2153         |     |
2154         |  4  |
2155         |     |
2156        19-----13
2157 
2158  and up through the top
2159 
2160        18-----16
2161         |     |
2162         |  2  |
2163         |     |
2164  18----10----11-----16
2165   |     |     |     |
2166   |  3  |  0  |  1  |
2167   |     |     |     |
2168  20-----9----12-----15
2169         |     |
2170         |  4  |
2171         |     |
2172        20-----15
2173 .ve
2174 
2175 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2176 @*/
2177 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, DMBoundaryType periodicZ, DM *dm)
2178 {
2179   PetscFunctionBegin;
2180   PetscValidPointer(dm, 3);
2181   PetscCall(DMCreate(comm, dm));
2182   PetscCall(DMSetType(*dm, DMPLEX));
2183   PetscCall(DMPlexCreateHexCylinderMesh_Internal(*dm, periodicZ));
2184   PetscFunctionReturn(0);
2185 }
2186 
2187 static PetscErrorCode DMPlexCreateWedgeCylinderMesh_Internal(DM dm, PetscInt n, PetscBool interpolate)
2188 {
2189   const PetscInt dim = 3;
2190   PetscInt       numCells, numVertices, v;
2191   PetscMPIInt    rank;
2192 
2193   PetscFunctionBegin;
2194   PetscCheck(n >= 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %" PetscInt_FMT " cannot be negative", n);
2195   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
2196   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2197   PetscCall(DMSetDimension(dm, dim));
2198   /* Must create the celltype label here so that we do not automatically try to compute the types */
2199   PetscCall(DMCreateLabel(dm, "celltype"));
2200   /* Create topology */
2201   {
2202     PetscInt cone[6], c;
2203 
2204     numCells    = rank == 0 ? n : 0;
2205     numVertices = rank == 0 ? 2 * (n + 1) : 0;
2206     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
2207     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
2208     PetscCall(DMSetUp(dm));
2209     for (c = 0; c < numCells; c++) {
2210       cone[0] = c + n * 1;
2211       cone[1] = (c + 1) % n + n * 1;
2212       cone[2] = 0 + 3 * n;
2213       cone[3] = c + n * 2;
2214       cone[4] = (c + 1) % n + n * 2;
2215       cone[5] = 1 + 3 * n;
2216       PetscCall(DMPlexSetCone(dm, c, cone));
2217       PetscCall(DMPlexSetCellType(dm, c, DM_POLYTOPE_TRI_PRISM_TENSOR));
2218     }
2219     PetscCall(DMPlexSymmetrize(dm));
2220     PetscCall(DMPlexStratify(dm));
2221   }
2222   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetCellType(dm, v, DM_POLYTOPE_POINT));
2223   /* Create cylinder geometry */
2224   {
2225     Vec          coordinates;
2226     PetscSection coordSection;
2227     PetscScalar *coords;
2228     PetscInt     coordSize, c;
2229 
2230     /* Build coordinates */
2231     PetscCall(DMGetCoordinateSection(dm, &coordSection));
2232     PetscCall(PetscSectionSetNumFields(coordSection, 1));
2233     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
2234     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
2235     for (v = numCells; v < numCells + numVertices; ++v) {
2236       PetscCall(PetscSectionSetDof(coordSection, v, dim));
2237       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
2238     }
2239     PetscCall(PetscSectionSetUp(coordSection));
2240     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2241     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2242     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2243     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2244     PetscCall(VecSetBlockSize(coordinates, dim));
2245     PetscCall(VecSetType(coordinates, VECSTANDARD));
2246     PetscCall(VecGetArray(coordinates, &coords));
2247     for (c = 0; c < numCells; c++) {
2248       coords[(c + 0 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
2249       coords[(c + 0 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
2250       coords[(c + 0 * n) * dim + 2] = 1.0;
2251       coords[(c + 1 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
2252       coords[(c + 1 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
2253       coords[(c + 1 * n) * dim + 2] = 0.0;
2254     }
2255     if (rank == 0) {
2256       coords[(2 * n + 0) * dim + 0] = 0.0;
2257       coords[(2 * n + 0) * dim + 1] = 0.0;
2258       coords[(2 * n + 0) * dim + 2] = 1.0;
2259       coords[(2 * n + 1) * dim + 0] = 0.0;
2260       coords[(2 * n + 1) * dim + 1] = 0.0;
2261       coords[(2 * n + 1) * dim + 2] = 0.0;
2262     }
2263     PetscCall(VecRestoreArray(coordinates, &coords));
2264     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2265     PetscCall(VecDestroy(&coordinates));
2266   }
2267   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
2268   /* Interpolate */
2269   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2270   PetscFunctionReturn(0);
2271 }
2272 
2273 /*@
2274   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
2275 
2276   Collective
2277 
2278   Input Parameters:
2279 + comm - The communicator for the `DM` object
2280 . n    - The number of wedges around the origin
2281 - interpolate - Create edges and faces
2282 
2283   Output Parameter:
2284 . dm  - The `DM` object
2285 
2286   Level: beginner
2287 
2288 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2289 @*/
2290 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
2291 {
2292   PetscFunctionBegin;
2293   PetscValidPointer(dm, 4);
2294   PetscCall(DMCreate(comm, dm));
2295   PetscCall(DMSetType(*dm, DMPLEX));
2296   PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(*dm, n, interpolate));
2297   PetscFunctionReturn(0);
2298 }
2299 
2300 static inline PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2301 {
2302   PetscReal prod = 0.0;
2303   PetscInt  i;
2304   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
2305   return PetscSqrtReal(prod);
2306 }
2307 static inline PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2308 {
2309   PetscReal prod = 0.0;
2310   PetscInt  i;
2311   for (i = 0; i < dim; ++i) prod += x[i] * y[i];
2312   return prod;
2313 }
2314 
2315 /* The first constant is the sphere radius */
2316 static void snapToSphere(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
2317 {
2318   PetscReal r     = PetscRealPart(constants[0]);
2319   PetscReal norm2 = 0.0, fac;
2320   PetscInt  n     = uOff[1] - uOff[0], d;
2321 
2322   for (d = 0; d < n; ++d) norm2 += PetscSqr(PetscRealPart(u[d]));
2323   fac = r / PetscSqrtReal(norm2);
2324   for (d = 0; d < n; ++d) f0[d] = u[d] * fac;
2325 }
2326 
2327 static PetscErrorCode DMPlexCreateSphereMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, PetscReal R)
2328 {
2329   const PetscInt embedDim = dim + 1;
2330   PetscSection   coordSection;
2331   Vec            coordinates;
2332   PetscScalar   *coords;
2333   PetscReal     *coordsIn;
2334   PetscInt       numCells, numEdges, numVerts = 0, firstVertex = 0, v, firstEdge, coordSize, d, c, e;
2335   PetscMPIInt    rank;
2336 
2337   PetscFunctionBegin;
2338   PetscValidLogicalCollectiveBool(dm, simplex, 3);
2339   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
2340   PetscCall(DMSetDimension(dm, dim));
2341   PetscCall(DMSetCoordinateDim(dm, dim + 1));
2342   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2343   switch (dim) {
2344   case 2:
2345     if (simplex) {
2346       const PetscReal radius    = PetscSqrtReal(1 + PETSC_PHI * PETSC_PHI) / (1.0 + PETSC_PHI);
2347       const PetscReal edgeLen   = 2.0 / (1.0 + PETSC_PHI) * (R / radius);
2348       const PetscInt  degree    = 5;
2349       PetscReal       vertex[3] = {0.0, 1.0 / (1.0 + PETSC_PHI), PETSC_PHI / (1.0 + PETSC_PHI)};
2350       PetscInt        s[3]      = {1, 1, 1};
2351       PetscInt        cone[3];
2352       PetscInt       *graph, p, i, j, k;
2353 
2354       vertex[0] *= R / radius;
2355       vertex[1] *= R / radius;
2356       vertex[2] *= R / radius;
2357       numCells    = rank == 0 ? 20 : 0;
2358       numVerts    = rank == 0 ? 12 : 0;
2359       firstVertex = numCells;
2360       /* Use icosahedron, which for a R-sphere has coordinates which are all cyclic permutations of
2361 
2362            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
2363 
2364          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2365          length is then given by 2/(1+\phi) = 2 * 0.38197 = 0.76393.
2366       */
2367       /* Construct vertices */
2368       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2369       if (rank == 0) {
2370         for (p = 0, i = 0; p < embedDim; ++p) {
2371           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2372             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2373               for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertex[(d + p) % embedDim];
2374               ++i;
2375             }
2376           }
2377         }
2378       }
2379       /* Construct graph */
2380       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2381       for (i = 0; i < numVerts; ++i) {
2382         for (j = 0, k = 0; j < numVerts; ++j) {
2383           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2384             graph[i * numVerts + j] = 1;
2385             ++k;
2386           }
2387         }
2388         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid icosahedron, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2389       }
2390       /* Build Topology */
2391       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2392       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2393       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2394       /* Cells */
2395       for (i = 0, c = 0; i < numVerts; ++i) {
2396         for (j = 0; j < i; ++j) {
2397           for (k = 0; k < j; ++k) {
2398             if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i]) {
2399               cone[0] = firstVertex + i;
2400               cone[1] = firstVertex + j;
2401               cone[2] = firstVertex + k;
2402               /* Check orientation */
2403               {
2404                 const PetscInt epsilon[3][3][3] = {
2405                   {{0, 0, 0},  {0, 0, 1},  {0, -1, 0}},
2406                   {{0, 0, -1}, {0, 0, 0},  {1, 0, 0} },
2407                   {{0, 1, 0},  {-1, 0, 0}, {0, 0, 0} }
2408                 };
2409                 PetscReal normal[3];
2410                 PetscInt  e, f;
2411 
2412                 for (d = 0; d < embedDim; ++d) {
2413                   normal[d] = 0.0;
2414                   for (e = 0; e < embedDim; ++e) {
2415                     for (f = 0; f < embedDim; ++f) normal[d] += epsilon[d][e][f] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]);
2416                   }
2417                 }
2418                 if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2419                   PetscInt tmp = cone[1];
2420                   cone[1]      = cone[2];
2421                   cone[2]      = tmp;
2422                 }
2423               }
2424               PetscCall(DMPlexSetCone(dm, c++, cone));
2425             }
2426           }
2427         }
2428       }
2429       PetscCall(DMPlexSymmetrize(dm));
2430       PetscCall(DMPlexStratify(dm));
2431       PetscCall(PetscFree(graph));
2432     } else {
2433       /*
2434         12-21--13
2435          |     |
2436         25  4  24
2437          |     |
2438   12-25--9-16--8-24--13
2439    |     |     |     |
2440   23  5 17  0 15  3  22
2441    |     |     |     |
2442   10-20--6-14--7-19--11
2443          |     |
2444         20  1  19
2445          |     |
2446         10-18--11
2447          |     |
2448         23  2  22
2449          |     |
2450         12-21--13
2451        */
2452       PetscInt cone[4], ornt[4];
2453 
2454       numCells    = rank == 0 ? 6 : 0;
2455       numEdges    = rank == 0 ? 12 : 0;
2456       numVerts    = rank == 0 ? 8 : 0;
2457       firstVertex = numCells;
2458       firstEdge   = numCells + numVerts;
2459       /* Build Topology */
2460       PetscCall(DMPlexSetChart(dm, 0, numCells + numEdges + numVerts));
2461       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 4));
2462       for (e = firstEdge; e < firstEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
2463       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2464       if (rank == 0) {
2465         /* Cell 0 */
2466         cone[0] = 14;
2467         cone[1] = 15;
2468         cone[2] = 16;
2469         cone[3] = 17;
2470         PetscCall(DMPlexSetCone(dm, 0, cone));
2471         ornt[0] = 0;
2472         ornt[1] = 0;
2473         ornt[2] = 0;
2474         ornt[3] = 0;
2475         PetscCall(DMPlexSetConeOrientation(dm, 0, ornt));
2476         /* Cell 1 */
2477         cone[0] = 18;
2478         cone[1] = 19;
2479         cone[2] = 14;
2480         cone[3] = 20;
2481         PetscCall(DMPlexSetCone(dm, 1, cone));
2482         ornt[0] = 0;
2483         ornt[1] = 0;
2484         ornt[2] = -1;
2485         ornt[3] = 0;
2486         PetscCall(DMPlexSetConeOrientation(dm, 1, ornt));
2487         /* Cell 2 */
2488         cone[0] = 21;
2489         cone[1] = 22;
2490         cone[2] = 18;
2491         cone[3] = 23;
2492         PetscCall(DMPlexSetCone(dm, 2, cone));
2493         ornt[0] = 0;
2494         ornt[1] = 0;
2495         ornt[2] = -1;
2496         ornt[3] = 0;
2497         PetscCall(DMPlexSetConeOrientation(dm, 2, ornt));
2498         /* Cell 3 */
2499         cone[0] = 19;
2500         cone[1] = 22;
2501         cone[2] = 24;
2502         cone[3] = 15;
2503         PetscCall(DMPlexSetCone(dm, 3, cone));
2504         ornt[0] = -1;
2505         ornt[1] = -1;
2506         ornt[2] = 0;
2507         ornt[3] = -1;
2508         PetscCall(DMPlexSetConeOrientation(dm, 3, ornt));
2509         /* Cell 4 */
2510         cone[0] = 16;
2511         cone[1] = 24;
2512         cone[2] = 21;
2513         cone[3] = 25;
2514         PetscCall(DMPlexSetCone(dm, 4, cone));
2515         ornt[0] = -1;
2516         ornt[1] = -1;
2517         ornt[2] = -1;
2518         ornt[3] = 0;
2519         PetscCall(DMPlexSetConeOrientation(dm, 4, ornt));
2520         /* Cell 5 */
2521         cone[0] = 20;
2522         cone[1] = 17;
2523         cone[2] = 25;
2524         cone[3] = 23;
2525         PetscCall(DMPlexSetCone(dm, 5, cone));
2526         ornt[0] = -1;
2527         ornt[1] = -1;
2528         ornt[2] = -1;
2529         ornt[3] = -1;
2530         PetscCall(DMPlexSetConeOrientation(dm, 5, ornt));
2531         /* Edges */
2532         cone[0] = 6;
2533         cone[1] = 7;
2534         PetscCall(DMPlexSetCone(dm, 14, cone));
2535         cone[0] = 7;
2536         cone[1] = 8;
2537         PetscCall(DMPlexSetCone(dm, 15, cone));
2538         cone[0] = 8;
2539         cone[1] = 9;
2540         PetscCall(DMPlexSetCone(dm, 16, cone));
2541         cone[0] = 9;
2542         cone[1] = 6;
2543         PetscCall(DMPlexSetCone(dm, 17, cone));
2544         cone[0] = 10;
2545         cone[1] = 11;
2546         PetscCall(DMPlexSetCone(dm, 18, cone));
2547         cone[0] = 11;
2548         cone[1] = 7;
2549         PetscCall(DMPlexSetCone(dm, 19, cone));
2550         cone[0] = 6;
2551         cone[1] = 10;
2552         PetscCall(DMPlexSetCone(dm, 20, cone));
2553         cone[0] = 12;
2554         cone[1] = 13;
2555         PetscCall(DMPlexSetCone(dm, 21, cone));
2556         cone[0] = 13;
2557         cone[1] = 11;
2558         PetscCall(DMPlexSetCone(dm, 22, cone));
2559         cone[0] = 10;
2560         cone[1] = 12;
2561         PetscCall(DMPlexSetCone(dm, 23, cone));
2562         cone[0] = 13;
2563         cone[1] = 8;
2564         PetscCall(DMPlexSetCone(dm, 24, cone));
2565         cone[0] = 12;
2566         cone[1] = 9;
2567         PetscCall(DMPlexSetCone(dm, 25, cone));
2568       }
2569       PetscCall(DMPlexSymmetrize(dm));
2570       PetscCall(DMPlexStratify(dm));
2571       /* Build coordinates */
2572       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2573       if (rank == 0) {
2574         coordsIn[0 * embedDim + 0] = -R;
2575         coordsIn[0 * embedDim + 1] = R;
2576         coordsIn[0 * embedDim + 2] = -R;
2577         coordsIn[1 * embedDim + 0] = R;
2578         coordsIn[1 * embedDim + 1] = R;
2579         coordsIn[1 * embedDim + 2] = -R;
2580         coordsIn[2 * embedDim + 0] = R;
2581         coordsIn[2 * embedDim + 1] = -R;
2582         coordsIn[2 * embedDim + 2] = -R;
2583         coordsIn[3 * embedDim + 0] = -R;
2584         coordsIn[3 * embedDim + 1] = -R;
2585         coordsIn[3 * embedDim + 2] = -R;
2586         coordsIn[4 * embedDim + 0] = -R;
2587         coordsIn[4 * embedDim + 1] = R;
2588         coordsIn[4 * embedDim + 2] = R;
2589         coordsIn[5 * embedDim + 0] = R;
2590         coordsIn[5 * embedDim + 1] = R;
2591         coordsIn[5 * embedDim + 2] = R;
2592         coordsIn[6 * embedDim + 0] = -R;
2593         coordsIn[6 * embedDim + 1] = -R;
2594         coordsIn[6 * embedDim + 2] = R;
2595         coordsIn[7 * embedDim + 0] = R;
2596         coordsIn[7 * embedDim + 1] = -R;
2597         coordsIn[7 * embedDim + 2] = R;
2598       }
2599     }
2600     break;
2601   case 3:
2602     if (simplex) {
2603       const PetscReal edgeLen         = 1.0 / PETSC_PHI;
2604       PetscReal       vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
2605       PetscReal       vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
2606       PetscReal       vertexC[4]      = {0.5, 0.5 * PETSC_PHI, 0.5 / PETSC_PHI, 0.0};
2607       const PetscInt  degree          = 12;
2608       PetscInt        s[4]            = {1, 1, 1};
2609       PetscInt        evenPerm[12][4] = {
2610         {0, 1, 2, 3},
2611         {0, 2, 3, 1},
2612         {0, 3, 1, 2},
2613         {1, 0, 3, 2},
2614         {1, 2, 0, 3},
2615         {1, 3, 2, 0},
2616         {2, 0, 1, 3},
2617         {2, 1, 3, 0},
2618         {2, 3, 0, 1},
2619         {3, 0, 2, 1},
2620         {3, 1, 0, 2},
2621         {3, 2, 1, 0}
2622       };
2623       PetscInt  cone[4];
2624       PetscInt *graph, p, i, j, k, l;
2625 
2626       vertexA[0] *= R;
2627       vertexA[1] *= R;
2628       vertexA[2] *= R;
2629       vertexA[3] *= R;
2630       vertexB[0] *= R;
2631       vertexB[1] *= R;
2632       vertexB[2] *= R;
2633       vertexB[3] *= R;
2634       vertexC[0] *= R;
2635       vertexC[1] *= R;
2636       vertexC[2] *= R;
2637       vertexC[3] *= R;
2638       numCells    = rank == 0 ? 600 : 0;
2639       numVerts    = rank == 0 ? 120 : 0;
2640       firstVertex = numCells;
2641       /* Use the 600-cell, which for a unit sphere has coordinates which are
2642 
2643            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
2644                (\pm 1,    0,       0,      0)  all cyclic permutations   8
2645            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
2646 
2647          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2648          length is then given by 1/\phi = 0.61803.
2649 
2650          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
2651          http://mathworld.wolfram.com/600-Cell.html
2652       */
2653       /* Construct vertices */
2654       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2655       i = 0;
2656       if (rank == 0) {
2657         for (s[0] = -1; s[0] < 2; s[0] += 2) {
2658           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2659             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2660               for (s[3] = -1; s[3] < 2; s[3] += 2) {
2661                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[d] * vertexA[d];
2662                 ++i;
2663               }
2664             }
2665           }
2666         }
2667         for (p = 0; p < embedDim; ++p) {
2668           s[1] = s[2] = s[3] = 1;
2669           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2670             for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertexB[(d + p) % embedDim];
2671             ++i;
2672           }
2673         }
2674         for (p = 0; p < 12; ++p) {
2675           s[3] = 1;
2676           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2677             for (s[1] = -1; s[1] < 2; s[1] += 2) {
2678               for (s[2] = -1; s[2] < 2; s[2] += 2) {
2679                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[evenPerm[p][d]] * vertexC[evenPerm[p][d]];
2680                 ++i;
2681               }
2682             }
2683           }
2684         }
2685       }
2686       PetscCheck(i == numVerts, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertices %" PetscInt_FMT " != %" PetscInt_FMT, i, numVerts);
2687       /* Construct graph */
2688       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2689       for (i = 0; i < numVerts; ++i) {
2690         for (j = 0, k = 0; j < numVerts; ++j) {
2691           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2692             graph[i * numVerts + j] = 1;
2693             ++k;
2694           }
2695         }
2696         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2697       }
2698       /* Build Topology */
2699       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2700       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2701       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2702       /* Cells */
2703       if (rank == 0) {
2704         for (i = 0, c = 0; i < numVerts; ++i) {
2705           for (j = 0; j < i; ++j) {
2706             for (k = 0; k < j; ++k) {
2707               for (l = 0; l < k; ++l) {
2708                 if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i] && graph[l * numVerts + i] && graph[l * numVerts + j] && graph[l * numVerts + k]) {
2709                   cone[0] = firstVertex + i;
2710                   cone[1] = firstVertex + j;
2711                   cone[2] = firstVertex + k;
2712                   cone[3] = firstVertex + l;
2713                   /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
2714                   {
2715                     const PetscInt epsilon[4][4][4][4] = {
2716                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, -1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 0, 0}, {0, 1, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 1, 0}, {0, -1, 0, 0}, {0, 0, 0, 0}}},
2717 
2718                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}}, {{0, 0, -1, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}}},
2719 
2720                       {{{0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, 0, 0}, {0, -1, 0, 0}}, {{0, 0, 0, -1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 1, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}},
2721 
2722                       {{{0, 0, 0, 0}, {0, 0, -1, 0}, {0, 1, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 1, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}}, {{0, -1, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}} }
2723                     };
2724                     PetscReal normal[4];
2725                     PetscInt  e, f, g;
2726 
2727                     for (d = 0; d < embedDim; ++d) {
2728                       normal[d] = 0.0;
2729                       for (e = 0; e < embedDim; ++e) {
2730                         for (f = 0; f < embedDim; ++f) {
2731                           for (g = 0; g < embedDim; ++g) {
2732                             normal[d] += epsilon[d][e][f][g] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]) * (coordsIn[l * embedDim + f] - coordsIn[i * embedDim + f]);
2733                           }
2734                         }
2735                       }
2736                     }
2737                     if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2738                       PetscInt tmp = cone[1];
2739                       cone[1]      = cone[2];
2740                       cone[2]      = tmp;
2741                     }
2742                   }
2743                   PetscCall(DMPlexSetCone(dm, c++, cone));
2744                 }
2745               }
2746             }
2747           }
2748         }
2749       }
2750       PetscCall(DMPlexSymmetrize(dm));
2751       PetscCall(DMPlexStratify(dm));
2752       PetscCall(PetscFree(graph));
2753     }
2754     break;
2755   default:
2756     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unsupported dimension for sphere: %" PetscInt_FMT, dim);
2757   }
2758   /* Create coordinates */
2759   PetscCall(DMGetCoordinateSection(dm, &coordSection));
2760   PetscCall(PetscSectionSetNumFields(coordSection, 1));
2761   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, embedDim));
2762   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVerts));
2763   for (v = firstVertex; v < firstVertex + numVerts; ++v) {
2764     PetscCall(PetscSectionSetDof(coordSection, v, embedDim));
2765     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, embedDim));
2766   }
2767   PetscCall(PetscSectionSetUp(coordSection));
2768   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2769   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2770   PetscCall(VecSetBlockSize(coordinates, embedDim));
2771   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2772   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2773   PetscCall(VecSetType(coordinates, VECSTANDARD));
2774   PetscCall(VecGetArray(coordinates, &coords));
2775   for (v = 0; v < numVerts; ++v)
2776     for (d = 0; d < embedDim; ++d) coords[v * embedDim + d] = coordsIn[v * embedDim + d];
2777   PetscCall(VecRestoreArray(coordinates, &coords));
2778   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2779   PetscCall(VecDestroy(&coordinates));
2780   PetscCall(PetscFree(coordsIn));
2781   {
2782     DM          cdm;
2783     PetscDS     cds;
2784     PetscScalar c = R;
2785 
2786     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToSphere));
2787     PetscCall(DMGetCoordinateDM(dm, &cdm));
2788     PetscCall(DMGetDS(cdm, &cds));
2789     PetscCall(PetscDSSetConstants(cds, 1, &c));
2790   }
2791   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
2792   /* Wait for coordinate creation before doing in-place modification */
2793   if (simplex) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2794   PetscFunctionReturn(0);
2795 }
2796 
2797 typedef void (*TPSEvaluateFunc)(const PetscReal[], PetscReal *, PetscReal[], PetscReal (*)[3]);
2798 
2799 /*
2800  The Schwarz P implicit surface is
2801 
2802      f(x) = cos(x0) + cos(x1) + cos(x2) = 0
2803 */
2804 static void TPSEvaluate_SchwarzP(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2805 {
2806   PetscReal c[3] = {PetscCosReal(y[0] * PETSC_PI), PetscCosReal(y[1] * PETSC_PI), PetscCosReal(y[2] * PETSC_PI)};
2807   PetscReal g[3] = {-PetscSinReal(y[0] * PETSC_PI), -PetscSinReal(y[1] * PETSC_PI), -PetscSinReal(y[2] * PETSC_PI)};
2808   f[0]           = c[0] + c[1] + c[2];
2809   for (PetscInt i = 0; i < 3; i++) {
2810     grad[i] = PETSC_PI * g[i];
2811     for (PetscInt j = 0; j < 3; j++) hess[i][j] = (i == j) ? -PetscSqr(PETSC_PI) * c[i] : 0.;
2812   }
2813 }
2814 
2815 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2816 static PetscErrorCode TPSExtrudeNormalFunc_SchwarzP(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2817 {
2818   for (PetscInt i = 0; i < 3; i++) u[i] = -PETSC_PI * PetscSinReal(x[i] * PETSC_PI);
2819   return 0;
2820 }
2821 
2822 /*
2823  The Gyroid implicit surface is
2824 
2825  f(x,y,z) = sin(pi * x) * cos (pi * (y + 1/2))  + sin(pi * (y + 1/2)) * cos(pi * (z + 1/4)) + sin(pi * (z + 1/4)) * cos(pi * x)
2826 
2827 */
2828 static void TPSEvaluate_Gyroid(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2829 {
2830   PetscReal s[3] = {PetscSinReal(PETSC_PI * y[0]), PetscSinReal(PETSC_PI * (y[1] + .5)), PetscSinReal(PETSC_PI * (y[2] + .25))};
2831   PetscReal c[3] = {PetscCosReal(PETSC_PI * y[0]), PetscCosReal(PETSC_PI * (y[1] + .5)), PetscCosReal(PETSC_PI * (y[2] + .25))};
2832   f[0]           = s[0] * c[1] + s[1] * c[2] + s[2] * c[0];
2833   grad[0]        = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2834   grad[1]        = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2835   grad[2]        = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2836   hess[0][0]     = -PetscSqr(PETSC_PI) * (s[0] * c[1] + s[2] * c[0]);
2837   hess[0][1]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2838   hess[0][2]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2839   hess[1][0]     = -PetscSqr(PETSC_PI) * (s[1] * c[2] + s[0] * c[1]);
2840   hess[1][1]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2841   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2842   hess[2][0]     = -PetscSqr(PETSC_PI) * (s[2] * c[0] + s[1] * c[2]);
2843   hess[2][1]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2844   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2845 }
2846 
2847 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2848 static PetscErrorCode TPSExtrudeNormalFunc_Gyroid(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2849 {
2850   PetscReal s[3] = {PetscSinReal(PETSC_PI * x[0]), PetscSinReal(PETSC_PI * (x[1] + .5)), PetscSinReal(PETSC_PI * (x[2] + .25))};
2851   PetscReal c[3] = {PetscCosReal(PETSC_PI * x[0]), PetscCosReal(PETSC_PI * (x[1] + .5)), PetscCosReal(PETSC_PI * (x[2] + .25))};
2852   u[0]           = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2853   u[1]           = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2854   u[2]           = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2855   return 0;
2856 }
2857 
2858 /*
2859    We wish to solve
2860 
2861          min_y || y - x ||^2  subject to f(y) = 0
2862 
2863    Let g(y) = grad(f).  The minimization problem is equivalent to asking to satisfy
2864    f(y) = 0 and (y-x) is parallel to g(y).  We do this by using Householder QR to obtain a basis for the
2865    tangent space and ask for both components in the tangent space to be zero.
2866 
2867    Take g to be a column vector and compute the "full QR" factorization Q R = g,
2868    where Q = I - 2 n n^T is a symmetric orthogonal matrix.
2869    The first column of Q is parallel to g so the remaining two columns span the null space.
2870    Let Qn = Q[:,1:] be those remaining columns.  Then Qn Qn^T is an orthogonal projector into the tangent space.
2871    Since Q is symmetric, this is equivalent to multipyling by Q and taking the last two entries.
2872    In total, we have a system of 3 equations in 3 unknowns:
2873 
2874      f(y) = 0                       1 equation
2875      Qn^T (y - x) = 0               2 equations
2876 
2877    Here, we compute the residual and Jacobian of this system.
2878 */
2879 static void TPSNearestPointResJac(TPSEvaluateFunc feval, const PetscScalar x[], const PetscScalar y[], PetscScalar res[], PetscScalar J[])
2880 {
2881   PetscReal yreal[3] = {PetscRealPart(y[0]), PetscRealPart(y[1]), PetscRealPart(y[2])};
2882   PetscReal d[3]     = {PetscRealPart(y[0] - x[0]), PetscRealPart(y[1] - x[1]), PetscRealPart(y[2] - x[2])};
2883   PetscReal f, grad[3], n[3], norm, norm_y[3], nd, nd_y[3], sign;
2884   PetscReal n_y[3][3] = {
2885     {0, 0, 0},
2886     {0, 0, 0},
2887     {0, 0, 0}
2888   };
2889 
2890   feval(yreal, &f, grad, n_y);
2891 
2892   for (PetscInt i = 0; i < 3; i++) n[i] = grad[i];
2893   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2894   for (PetscInt i = 0; i < 3; i++) norm_y[i] = 1. / norm * n[i] * n_y[i][i];
2895 
2896   // Define the Householder reflector
2897   sign = n[0] >= 0 ? 1. : -1.;
2898   n[0] += norm * sign;
2899   for (PetscInt i = 0; i < 3; i++) n_y[0][i] += norm_y[i] * sign;
2900 
2901   norm      = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2902   norm_y[0] = 1. / norm * (n[0] * n_y[0][0]);
2903   norm_y[1] = 1. / norm * (n[0] * n_y[0][1] + n[1] * n_y[1][1]);
2904   norm_y[2] = 1. / norm * (n[0] * n_y[0][2] + n[2] * n_y[2][2]);
2905 
2906   for (PetscInt i = 0; i < 3; i++) {
2907     n[i] /= norm;
2908     for (PetscInt j = 0; j < 3; j++) {
2909       // note that n[i] is n_old[i]/norm when executing the code below
2910       n_y[i][j] = n_y[i][j] / norm - n[i] / norm * norm_y[j];
2911     }
2912   }
2913 
2914   nd = n[0] * d[0] + n[1] * d[1] + n[2] * d[2];
2915   for (PetscInt i = 0; i < 3; i++) nd_y[i] = n[i] + n_y[0][i] * d[0] + n_y[1][i] * d[1] + n_y[2][i] * d[2];
2916 
2917   res[0] = f;
2918   res[1] = d[1] - 2 * n[1] * nd;
2919   res[2] = d[2] - 2 * n[2] * nd;
2920   // J[j][i] is J_{ij} (column major)
2921   for (PetscInt j = 0; j < 3; j++) {
2922     J[0 + j * 3] = grad[j];
2923     J[1 + j * 3] = (j == 1) * 1. - 2 * (n_y[1][j] * nd + n[1] * nd_y[j]);
2924     J[2 + j * 3] = (j == 2) * 1. - 2 * (n_y[2][j] * nd + n[2] * nd_y[j]);
2925   }
2926 }
2927 
2928 /*
2929    Project x to the nearest point on the implicit surface using Newton's method.
2930 */
2931 static PetscErrorCode TPSNearestPoint(TPSEvaluateFunc feval, PetscScalar x[])
2932 {
2933   PetscScalar y[3] = {x[0], x[1], x[2]}; // Initial guess
2934 
2935   PetscFunctionBegin;
2936   for (PetscInt iter = 0; iter < 10; iter++) {
2937     PetscScalar res[3], J[9];
2938     PetscReal   resnorm;
2939     TPSNearestPointResJac(feval, x, y, res, J);
2940     resnorm = PetscSqrtReal(PetscSqr(PetscRealPart(res[0])) + PetscSqr(PetscRealPart(res[1])) + PetscSqr(PetscRealPart(res[2])));
2941     if (0) { // Turn on this monitor if you need to confirm quadratic convergence
2942       PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%" PetscInt_FMT "] res [%g %g %g]\n", iter, (double)PetscRealPart(res[0]), (double)PetscRealPart(res[1]), (double)PetscRealPart(res[2])));
2943     }
2944     if (resnorm < PETSC_SMALL) break;
2945 
2946     // Take the Newton step
2947     PetscCall(PetscKernel_A_gets_inverse_A_3(J, 0., PETSC_FALSE, NULL));
2948     PetscKernel_v_gets_v_minus_A_times_w_3(y, J, res);
2949   }
2950   for (PetscInt i = 0; i < 3; i++) x[i] = y[i];
2951   PetscFunctionReturn(0);
2952 }
2953 
2954 const char *const DMPlexTPSTypes[] = {"SCHWARZ_P", "GYROID", "DMPlexTPSType", "DMPLEX_TPS_", NULL};
2955 
2956 static PetscErrorCode DMPlexCreateTPSMesh_Internal(DM dm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness)
2957 {
2958   PetscMPIInt rank;
2959   PetscInt    topoDim = 2, spaceDim = 3, numFaces = 0, numVertices = 0, numEdges = 0;
2960   PetscInt(*edges)[2] = NULL, *edgeSets = NULL;
2961   PetscInt            *cells_flat = NULL;
2962   PetscReal           *vtxCoords  = NULL;
2963   TPSEvaluateFunc      evalFunc   = NULL;
2964   PetscSimplePointFunc normalFunc = NULL;
2965   DMLabel              label;
2966 
2967   PetscFunctionBegin;
2968   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
2969   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2970   PetscCheck((layers != 0) ^ (thickness == 0.), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Layers %" PetscInt_FMT " must be nonzero iff thickness %g is nonzero", layers, (double)thickness);
2971   switch (tpstype) {
2972   case DMPLEX_TPS_SCHWARZ_P:
2973     PetscCheck(!periodic || (periodic[0] == DM_BOUNDARY_NONE && periodic[1] == DM_BOUNDARY_NONE && periodic[2] == DM_BOUNDARY_NONE), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Schwarz P does not support periodic meshes");
2974     if (rank == 0) {
2975       PetscInt(*cells)[6][4][4] = NULL; // [junction, junction-face, cell, conn]
2976       PetscInt  Njunctions = 0, Ncuts = 0, Npipes[3], vcount;
2977       PetscReal L = 1;
2978 
2979       Npipes[0]   = (extent[0] + 1) * extent[1] * extent[2];
2980       Npipes[1]   = extent[0] * (extent[1] + 1) * extent[2];
2981       Npipes[2]   = extent[0] * extent[1] * (extent[2] + 1);
2982       Njunctions  = extent[0] * extent[1] * extent[2];
2983       Ncuts       = 2 * (extent[0] * extent[1] + extent[1] * extent[2] + extent[2] * extent[0]);
2984       numVertices = 4 * (Npipes[0] + Npipes[1] + Npipes[2]) + 8 * Njunctions;
2985       PetscCall(PetscMalloc1(3 * numVertices, &vtxCoords));
2986       PetscCall(PetscMalloc1(Njunctions, &cells));
2987       PetscCall(PetscMalloc1(Ncuts * 4, &edges));
2988       PetscCall(PetscMalloc1(Ncuts * 4, &edgeSets));
2989       // x-normal pipes
2990       vcount = 0;
2991       for (PetscInt i = 0; i < extent[0] + 1; i++) {
2992         for (PetscInt j = 0; j < extent[1]; j++) {
2993           for (PetscInt k = 0; k < extent[2]; k++) {
2994             for (PetscInt l = 0; l < 4; l++) {
2995               vtxCoords[vcount++] = (2 * i - 1) * L;
2996               vtxCoords[vcount++] = 2 * j * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2997               vtxCoords[vcount++] = 2 * k * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
2998             }
2999           }
3000         }
3001       }
3002       // y-normal pipes
3003       for (PetscInt i = 0; i < extent[0]; i++) {
3004         for (PetscInt j = 0; j < extent[1] + 1; j++) {
3005           for (PetscInt k = 0; k < extent[2]; k++) {
3006             for (PetscInt l = 0; l < 4; l++) {
3007               vtxCoords[vcount++] = 2 * i * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3008               vtxCoords[vcount++] = (2 * j - 1) * L;
3009               vtxCoords[vcount++] = 2 * k * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3010             }
3011           }
3012         }
3013       }
3014       // z-normal pipes
3015       for (PetscInt i = 0; i < extent[0]; i++) {
3016         for (PetscInt j = 0; j < extent[1]; j++) {
3017           for (PetscInt k = 0; k < extent[2] + 1; k++) {
3018             for (PetscInt l = 0; l < 4; l++) {
3019               vtxCoords[vcount++] = 2 * i * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3020               vtxCoords[vcount++] = 2 * j * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3021               vtxCoords[vcount++] = (2 * k - 1) * L;
3022             }
3023           }
3024         }
3025       }
3026       // junctions
3027       for (PetscInt i = 0; i < extent[0]; i++) {
3028         for (PetscInt j = 0; j < extent[1]; j++) {
3029           for (PetscInt k = 0; k < extent[2]; k++) {
3030             const PetscInt J = (i * extent[1] + j) * extent[2] + k, Jvoff = (Npipes[0] + Npipes[1] + Npipes[2]) * 4 + J * 8;
3031             PetscCheck(vcount / 3 == Jvoff, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected vertex count");
3032             for (PetscInt ii = 0; ii < 2; ii++) {
3033               for (PetscInt jj = 0; jj < 2; jj++) {
3034                 for (PetscInt kk = 0; kk < 2; kk++) {
3035                   double Ls           = (1 - sqrt(2) / 4) * L;
3036                   vtxCoords[vcount++] = 2 * i * L + (2 * ii - 1) * Ls;
3037                   vtxCoords[vcount++] = 2 * j * L + (2 * jj - 1) * Ls;
3038                   vtxCoords[vcount++] = 2 * k * L + (2 * kk - 1) * Ls;
3039                 }
3040               }
3041             }
3042             const PetscInt jfaces[3][2][4] = {
3043               {{3, 1, 0, 2}, {7, 5, 4, 6}}, // x-aligned
3044               {{5, 4, 0, 1}, {7, 6, 2, 3}}, // y-aligned
3045               {{6, 2, 0, 4}, {7, 3, 1, 5}}  // z-aligned
3046             };
3047             const PetscInt pipe_lo[3] = {// vertex numbers of pipes
3048                                          ((i * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + Npipes[0] + Npipes[1]) * 4};
3049             const PetscInt pipe_hi[3] = {// vertex numbers of pipes
3050                                          (((i + 1) * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j + 1) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + 1 + Npipes[0] + Npipes[1]) * 4};
3051             for (PetscInt dir = 0; dir < 3; dir++) { // x,y,z
3052               const PetscInt ijk[3] = {i, j, k};
3053               for (PetscInt l = 0; l < 4; l++) { // rotations
3054                 cells[J][dir * 2 + 0][l][0] = pipe_lo[dir] + l;
3055                 cells[J][dir * 2 + 0][l][1] = Jvoff + jfaces[dir][0][l];
3056                 cells[J][dir * 2 + 0][l][2] = Jvoff + jfaces[dir][0][(l - 1 + 4) % 4];
3057                 cells[J][dir * 2 + 0][l][3] = pipe_lo[dir] + (l - 1 + 4) % 4;
3058                 cells[J][dir * 2 + 1][l][0] = Jvoff + jfaces[dir][1][l];
3059                 cells[J][dir * 2 + 1][l][1] = pipe_hi[dir] + l;
3060                 cells[J][dir * 2 + 1][l][2] = pipe_hi[dir] + (l - 1 + 4) % 4;
3061                 cells[J][dir * 2 + 1][l][3] = Jvoff + jfaces[dir][1][(l - 1 + 4) % 4];
3062                 if (ijk[dir] == 0) {
3063                   edges[numEdges][0] = pipe_lo[dir] + l;
3064                   edges[numEdges][1] = pipe_lo[dir] + (l + 1) % 4;
3065                   edgeSets[numEdges] = dir * 2 + 1;
3066                   numEdges++;
3067                 }
3068                 if (ijk[dir] + 1 == extent[dir]) {
3069                   edges[numEdges][0] = pipe_hi[dir] + l;
3070                   edges[numEdges][1] = pipe_hi[dir] + (l + 1) % 4;
3071                   edgeSets[numEdges] = dir * 2 + 2;
3072                   numEdges++;
3073                 }
3074               }
3075             }
3076           }
3077         }
3078       }
3079       PetscCheck(numEdges == Ncuts * 4, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Edge count %" PetscInt_FMT " incompatible with number of cuts %" PetscInt_FMT, numEdges, Ncuts);
3080       numFaces   = 24 * Njunctions;
3081       cells_flat = cells[0][0][0];
3082     }
3083     evalFunc   = TPSEvaluate_SchwarzP;
3084     normalFunc = TPSExtrudeNormalFunc_SchwarzP;
3085     break;
3086   case DMPLEX_TPS_GYROID:
3087     if (rank == 0) {
3088       // This is a coarse mesh approximation of the gyroid shifted to being the zero of the level set
3089       //
3090       //     sin(pi*x)*cos(pi*(y+1/2)) + sin(pi*(y+1/2))*cos(pi*(z+1/4)) + sin(pi*(z+1/4))*cos(x)
3091       //
3092       // on the cell [0,2]^3.
3093       //
3094       // Think about dividing that cell into four columns, and focus on the column [0,1]x[0,1]x[0,2].
3095       // If you looked at the gyroid in that column at different slices of z you would see that it kind of spins
3096       // like a boomerang:
3097       //
3098       //     z = 0          z = 1/4        z = 1/2        z = 3/4     //
3099       //     -----          -------        -------        -------     //
3100       //                                                              //
3101       //     +       +      +       +      +       +      +   \   +   //
3102       //      \                                   /            \      //
3103       //       \            `-_   _-'            /              }     //
3104       //        *-_            `-'            _-'              /      //
3105       //     +     `-+      +       +      +-'     +      +   /   +   //
3106       //                                                              //
3107       //                                                              //
3108       //     z = 1          z = 5/4        z = 3/2        z = 7/4     //
3109       //     -----          -------        -------        -------     //
3110       //                                                              //
3111       //     +-_     +      +       +      +     _-+      +   /   +   //
3112       //        `-_            _-_            _-`            /        //
3113       //           \        _-'   `-_        /              {         //
3114       //            \                       /                \        //
3115       //     +       +      +       +      +       +      +   \   +   //
3116       //
3117       //
3118       // This course mesh approximates each of these slices by two line segments,
3119       // and then connects the segments in consecutive layers with quadrilateral faces.
3120       // All of the end points of the segments are multiples of 1/4 except for the
3121       // point * in the picture for z = 0 above and the similar points in other layers.
3122       // That point is at (gamma, gamma, 0), where gamma is calculated below.
3123       //
3124       // The column  [1,2]x[1,2]x[0,2] looks the same as this column;
3125       // The columns [1,2]x[0,1]x[0,2] and [0,1]x[1,2]x[0,2] are mirror images.
3126       //
3127       // As for how this method turned into the names given to the vertices:
3128       // that was not systematic, it was just the way it worked out in my handwritten notes.
3129 
3130       PetscInt facesPerBlock = 64;
3131       PetscInt vertsPerBlock = 56;
3132       PetscInt extentPlus[3];
3133       PetscInt numBlocks, numBlocksPlus;
3134       const PetscInt A = 0, B = 1, C = 2, D = 3, E = 4, F = 5, G = 6, H = 7, II = 8, J = 9, K = 10, L = 11, M = 12, N = 13, O = 14, P = 15, Q = 16, R = 17, S = 18, T = 19, U = 20, V = 21, W = 22, X = 23, Y = 24, Z = 25, Ap = 26, Bp = 27, Cp = 28, Dp = 29, Ep = 30, Fp = 31, Gp = 32, Hp = 33, Ip = 34, Jp = 35, Kp = 36, Lp = 37, Mp = 38, Np = 39, Op = 40, Pp = 41, Qp = 42, Rp = 43, Sp = 44, Tp = 45, Up = 46, Vp = 47, Wp = 48, Xp = 49, Yp = 50, Zp = 51, Aq = 52, Bq = 53, Cq = 54, Dq = 55;
3135       const PetscInt pattern[64][4] = {
3136   /* face to vertex within the coarse discretization of a single gyroid block */
3137   /* layer 0 */
3138         {A,           C,           K,           G          },
3139         {C,           B,           II,          K          },
3140         {D,           A,           H,           L          },
3141         {B + 56 * 1,  D,           L,           J          },
3142         {E,           B + 56 * 1,  J,           N          },
3143         {A + 56 * 2,  E,           N,           H + 56 * 2 },
3144         {F,           A + 56 * 2,  G + 56 * 2,  M          },
3145         {B,           F,           M,           II         },
3146  /* layer 1 */
3147         {G,           K,           Q,           O          },
3148         {K,           II,          P,           Q          },
3149         {L,           H,           O + 56 * 1,  R          },
3150         {J,           L,           R,           P          },
3151         {N,           J,           P,           S          },
3152         {H + 56 * 2,  N,           S,           O + 56 * 3 },
3153         {M,           G + 56 * 2,  O + 56 * 2,  T          },
3154         {II,          M,           T,           P          },
3155  /* layer 2 */
3156         {O,           Q,           Y,           U          },
3157         {Q,           P,           W,           Y          },
3158         {R,           O + 56 * 1,  U + 56 * 1,  Ap         },
3159         {P,           R,           Ap,          W          },
3160         {S,           P,           X,           Bp         },
3161         {O + 56 * 3,  S,           Bp,          V + 56 * 1 },
3162         {T,           O + 56 * 2,  V,           Z          },
3163         {P,           T,           Z,           X          },
3164  /* layer 3 */
3165         {U,           Y,           Ep,          Dp         },
3166         {Y,           W,           Cp,          Ep         },
3167         {Ap,          U + 56 * 1,  Dp + 56 * 1, Gp         },
3168         {W,           Ap,          Gp,          Cp         },
3169         {Bp,          X,           Cp + 56 * 2, Fp         },
3170         {V + 56 * 1,  Bp,          Fp,          Dp + 56 * 1},
3171         {Z,           V,           Dp,          Hp         },
3172         {X,           Z,           Hp,          Cp + 56 * 2},
3173  /* layer 4 */
3174         {Dp,          Ep,          Mp,          Kp         },
3175         {Ep,          Cp,          Ip,          Mp         },
3176         {Gp,          Dp + 56 * 1, Lp,          Np         },
3177         {Cp,          Gp,          Np,          Jp         },
3178         {Fp,          Cp + 56 * 2, Jp + 56 * 2, Pp         },
3179         {Dp + 56 * 1, Fp,          Pp,          Lp         },
3180         {Hp,          Dp,          Kp,          Op         },
3181         {Cp + 56 * 2, Hp,          Op,          Ip + 56 * 2},
3182  /* layer 5 */
3183         {Kp,          Mp,          Sp,          Rp         },
3184         {Mp,          Ip,          Qp,          Sp         },
3185         {Np,          Lp,          Rp,          Tp         },
3186         {Jp,          Np,          Tp,          Qp + 56 * 1},
3187         {Pp,          Jp + 56 * 2, Qp + 56 * 3, Up         },
3188         {Lp,          Pp,          Up,          Rp         },
3189         {Op,          Kp,          Rp,          Vp         },
3190         {Ip + 56 * 2, Op,          Vp,          Qp + 56 * 2},
3191  /* layer 6 */
3192         {Rp,          Sp,          Aq,          Yp         },
3193         {Sp,          Qp,          Wp,          Aq         },
3194         {Tp,          Rp,          Yp,          Cq         },
3195         {Qp + 56 * 1, Tp,          Cq,          Wp + 56 * 1},
3196         {Up,          Qp + 56 * 3, Xp + 56 * 1, Dq         },
3197         {Rp,          Up,          Dq,          Zp         },
3198         {Vp,          Rp,          Zp,          Bq         },
3199         {Qp + 56 * 2, Vp,          Bq,          Xp         },
3200  /* layer 7 (the top is the periodic image of the bottom of layer 0) */
3201         {Yp,          Aq,          C + 56 * 4,  A + 56 * 4 },
3202         {Aq,          Wp,          B + 56 * 4,  C + 56 * 4 },
3203         {Cq,          Yp,          A + 56 * 4,  D + 56 * 4 },
3204         {Wp + 56 * 1, Cq,          D + 56 * 4,  B + 56 * 5 },
3205         {Dq,          Xp + 56 * 1, B + 56 * 5,  E + 56 * 4 },
3206         {Zp,          Dq,          E + 56 * 4,  A + 56 * 6 },
3207         {Bq,          Zp,          A + 56 * 6,  F + 56 * 4 },
3208         {Xp,          Bq,          F + 56 * 4,  B + 56 * 4 }
3209       };
3210       const PetscReal gamma                = PetscAcosReal((PetscSqrtReal(3.) - 1.) / PetscSqrtReal(2.)) / PETSC_PI;
3211       const PetscReal patternCoords[56][3] = {
3212         {1.,        0.,        0.  }, /* A  */
3213         {0.,        1.,        0.  }, /* B  */
3214         {gamma,     gamma,     0.  }, /* C  */
3215         {1 + gamma, 1 - gamma, 0.  }, /* D  */
3216         {2 - gamma, 2 - gamma, 0.  }, /* E  */
3217         {1 - gamma, 1 + gamma, 0.  }, /* F  */
3218 
3219         {.5,        0,         .25 }, /* G  */
3220         {1.5,       0.,        .25 }, /* H  */
3221         {.5,        1.,        .25 }, /* II */
3222         {1.5,       1.,        .25 }, /* J  */
3223         {.25,       .5,        .25 }, /* K  */
3224         {1.25,      .5,        .25 }, /* L  */
3225         {.75,       1.5,       .25 }, /* M  */
3226         {1.75,      1.5,       .25 }, /* N  */
3227 
3228         {0.,        0.,        .5  }, /* O  */
3229         {1.,        1.,        .5  }, /* P  */
3230         {gamma,     1 - gamma, .5  }, /* Q  */
3231         {1 + gamma, gamma,     .5  }, /* R  */
3232         {2 - gamma, 1 + gamma, .5  }, /* S  */
3233         {1 - gamma, 2 - gamma, .5  }, /* T  */
3234 
3235         {0.,        .5,        .75 }, /* U  */
3236         {0.,        1.5,       .75 }, /* V  */
3237         {1.,        .5,        .75 }, /* W  */
3238         {1.,        1.5,       .75 }, /* X  */
3239         {.5,        .75,       .75 }, /* Y  */
3240         {.5,        1.75,      .75 }, /* Z  */
3241         {1.5,       .25,       .75 }, /* Ap */
3242         {1.5,       1.25,      .75 }, /* Bp */
3243 
3244         {1.,        0.,        1.  }, /* Cp */
3245         {0.,        1.,        1.  }, /* Dp */
3246         {1 - gamma, 1 - gamma, 1.  }, /* Ep */
3247         {1 + gamma, 1 + gamma, 1.  }, /* Fp */
3248         {2 - gamma, gamma,     1.  }, /* Gp */
3249         {gamma,     2 - gamma, 1.  }, /* Hp */
3250 
3251         {.5,        0.,        1.25}, /* Ip */
3252         {1.5,       0.,        1.25}, /* Jp */
3253         {.5,        1.,        1.25}, /* Kp */
3254         {1.5,       1.,        1.25}, /* Lp */
3255         {.75,       .5,        1.25}, /* Mp */
3256         {1.75,      .5,        1.25}, /* Np */
3257         {.25,       1.5,       1.25}, /* Op */
3258         {1.25,      1.5,       1.25}, /* Pp */
3259 
3260         {0.,        0.,        1.5 }, /* Qp */
3261         {1.,        1.,        1.5 }, /* Rp */
3262         {1 - gamma, gamma,     1.5 }, /* Sp */
3263         {2 - gamma, 1 - gamma, 1.5 }, /* Tp */
3264         {1 + gamma, 2 - gamma, 1.5 }, /* Up */
3265         {gamma,     1 + gamma, 1.5 }, /* Vp */
3266 
3267         {0.,        .5,        1.75}, /* Wp */
3268         {0.,        1.5,       1.75}, /* Xp */
3269         {1.,        .5,        1.75}, /* Yp */
3270         {1.,        1.5,       1.75}, /* Zp */
3271         {.5,        .25,       1.75}, /* Aq */
3272         {.5,        1.25,      1.75}, /* Bq */
3273         {1.5,       .75,       1.75}, /* Cq */
3274         {1.5,       1.75,      1.75}, /* Dq */
3275       };
3276       PetscInt(*cells)[64][4] = NULL;
3277       PetscBool *seen;
3278       PetscInt  *vertToTrueVert;
3279       PetscInt   count;
3280 
3281       for (PetscInt i = 0; i < 3; i++) extentPlus[i] = extent[i] + 1;
3282       numBlocks = 1;
3283       for (PetscInt i = 0; i < 3; i++) numBlocks *= extent[i];
3284       numBlocksPlus = 1;
3285       for (PetscInt i = 0; i < 3; i++) numBlocksPlus *= extentPlus[i];
3286       numFaces = numBlocks * facesPerBlock;
3287       PetscCall(PetscMalloc1(numBlocks, &cells));
3288       PetscCall(PetscCalloc1(numBlocksPlus * vertsPerBlock, &seen));
3289       for (PetscInt k = 0; k < extent[2]; k++) {
3290         for (PetscInt j = 0; j < extent[1]; j++) {
3291           for (PetscInt i = 0; i < extent[0]; i++) {
3292             for (PetscInt f = 0; f < facesPerBlock; f++) {
3293               for (PetscInt v = 0; v < 4; v++) {
3294                 PetscInt vertRaw     = pattern[f][v];
3295                 PetscInt blockidx    = vertRaw / 56;
3296                 PetscInt patternvert = vertRaw % 56;
3297                 PetscInt xplus       = (blockidx & 1);
3298                 PetscInt yplus       = (blockidx & 2) >> 1;
3299                 PetscInt zplus       = (blockidx & 4) >> 2;
3300                 PetscInt zcoord      = (periodic && periodic[2] == DM_BOUNDARY_PERIODIC) ? ((k + zplus) % extent[2]) : (k + zplus);
3301                 PetscInt ycoord      = (periodic && periodic[1] == DM_BOUNDARY_PERIODIC) ? ((j + yplus) % extent[1]) : (j + yplus);
3302                 PetscInt xcoord      = (periodic && periodic[0] == DM_BOUNDARY_PERIODIC) ? ((i + xplus) % extent[0]) : (i + xplus);
3303                 PetscInt vert        = ((zcoord * extentPlus[1] + ycoord) * extentPlus[0] + xcoord) * 56 + patternvert;
3304 
3305                 cells[(k * extent[1] + j) * extent[0] + i][f][v] = vert;
3306                 seen[vert]                                       = PETSC_TRUE;
3307               }
3308             }
3309           }
3310         }
3311       }
3312       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++)
3313         if (seen[i]) numVertices++;
3314       count = 0;
3315       PetscCall(PetscMalloc1(numBlocksPlus * vertsPerBlock, &vertToTrueVert));
3316       PetscCall(PetscMalloc1(numVertices * 3, &vtxCoords));
3317       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) vertToTrueVert[i] = -1;
3318       for (PetscInt k = 0; k < extentPlus[2]; k++) {
3319         for (PetscInt j = 0; j < extentPlus[1]; j++) {
3320           for (PetscInt i = 0; i < extentPlus[0]; i++) {
3321             for (PetscInt v = 0; v < vertsPerBlock; v++) {
3322               PetscInt vIdx = ((k * extentPlus[1] + j) * extentPlus[0] + i) * vertsPerBlock + v;
3323 
3324               if (seen[vIdx]) {
3325                 PetscInt thisVert;
3326 
3327                 vertToTrueVert[vIdx] = thisVert = count++;
3328 
3329                 for (PetscInt d = 0; d < 3; d++) vtxCoords[3 * thisVert + d] = patternCoords[v][d];
3330                 vtxCoords[3 * thisVert + 0] += i * 2;
3331                 vtxCoords[3 * thisVert + 1] += j * 2;
3332                 vtxCoords[3 * thisVert + 2] += k * 2;
3333               }
3334             }
3335           }
3336         }
3337       }
3338       for (PetscInt i = 0; i < numBlocks; i++) {
3339         for (PetscInt f = 0; f < facesPerBlock; f++) {
3340           for (PetscInt v = 0; v < 4; v++) cells[i][f][v] = vertToTrueVert[cells[i][f][v]];
3341         }
3342       }
3343       PetscCall(PetscFree(vertToTrueVert));
3344       PetscCall(PetscFree(seen));
3345       cells_flat = cells[0][0];
3346       numEdges   = 0;
3347       for (PetscInt i = 0; i < numFaces; i++) {
3348         for (PetscInt e = 0; e < 4; e++) {
3349           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3350           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3351 
3352           for (PetscInt d = 0; d < 3; d++) {
3353             if (!periodic || periodic[0] != DM_BOUNDARY_PERIODIC) {
3354               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) numEdges++;
3355               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) numEdges++;
3356             }
3357           }
3358         }
3359       }
3360       PetscCall(PetscMalloc1(numEdges, &edges));
3361       PetscCall(PetscMalloc1(numEdges, &edgeSets));
3362       for (PetscInt edge = 0, i = 0; i < numFaces; i++) {
3363         for (PetscInt e = 0; e < 4; e++) {
3364           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3365           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3366 
3367           for (PetscInt d = 0; d < 3; d++) {
3368             if (!periodic || periodic[d] != DM_BOUNDARY_PERIODIC) {
3369               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) {
3370                 edges[edge][0]   = ev[0];
3371                 edges[edge][1]   = ev[1];
3372                 edgeSets[edge++] = 2 * d;
3373               }
3374               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) {
3375                 edges[edge][0]   = ev[0];
3376                 edges[edge][1]   = ev[1];
3377                 edgeSets[edge++] = 2 * d + 1;
3378               }
3379             }
3380           }
3381         }
3382       }
3383     }
3384     evalFunc   = TPSEvaluate_Gyroid;
3385     normalFunc = TPSExtrudeNormalFunc_Gyroid;
3386     break;
3387   }
3388 
3389   PetscCall(DMSetDimension(dm, topoDim));
3390   if (rank == 0) PetscCall(DMPlexBuildFromCellList(dm, numFaces, numVertices, 4, cells_flat));
3391   else PetscCall(DMPlexBuildFromCellList(dm, 0, 0, 0, NULL));
3392   PetscCall(PetscFree(cells_flat));
3393   {
3394     DM idm;
3395     PetscCall(DMPlexInterpolate(dm, &idm));
3396     PetscCall(DMPlexReplace_Internal(dm, &idm));
3397   }
3398   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, vtxCoords));
3399   else PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, NULL));
3400   PetscCall(PetscFree(vtxCoords));
3401 
3402   PetscCall(DMCreateLabel(dm, "Face Sets"));
3403   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3404   for (PetscInt e = 0; e < numEdges; e++) {
3405     PetscInt        njoin;
3406     const PetscInt *join, verts[] = {numFaces + edges[e][0], numFaces + edges[e][1]};
3407     PetscCall(DMPlexGetJoin(dm, 2, verts, &njoin, &join));
3408     PetscCheck(njoin == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected unique join of vertices %" PetscInt_FMT " and %" PetscInt_FMT, edges[e][0], edges[e][1]);
3409     PetscCall(DMLabelSetValue(label, join[0], edgeSets[e]));
3410     PetscCall(DMPlexRestoreJoin(dm, 2, verts, &njoin, &join));
3411   }
3412   PetscCall(PetscFree(edges));
3413   PetscCall(PetscFree(edgeSets));
3414   if (tps_distribute) {
3415     DM               pdm = NULL;
3416     PetscPartitioner part;
3417 
3418     PetscCall(DMPlexGetPartitioner(dm, &part));
3419     PetscCall(PetscPartitionerSetFromOptions(part));
3420     PetscCall(DMPlexDistribute(dm, 0, NULL, &pdm));
3421     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
3422     // Do not auto-distribute again
3423     PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
3424   }
3425 
3426   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3427   for (PetscInt refine = 0; refine < refinements; refine++) {
3428     PetscInt     m;
3429     DM           dmf;
3430     Vec          X;
3431     PetscScalar *x;
3432     PetscCall(DMRefine(dm, MPI_COMM_NULL, &dmf));
3433     PetscCall(DMPlexReplace_Internal(dm, &dmf));
3434 
3435     PetscCall(DMGetCoordinatesLocal(dm, &X));
3436     PetscCall(VecGetLocalSize(X, &m));
3437     PetscCall(VecGetArray(X, &x));
3438     for (PetscInt i = 0; i < m; i += 3) PetscCall(TPSNearestPoint(evalFunc, &x[i]));
3439     PetscCall(VecRestoreArray(X, &x));
3440   }
3441 
3442   // Face Sets has already been propagated to new vertices during refinement; this propagates to the initial vertices.
3443   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3444   PetscCall(DMPlexLabelComplete(dm, label));
3445 
3446   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
3447 
3448   if (thickness > 0) {
3449     DM              edm, cdm, ecdm;
3450     DMPlexTransform tr;
3451     const char     *prefix;
3452     PetscOptions    options;
3453     // Code from DMPlexExtrude
3454     PetscCall(DMPlexTransformCreate(PetscObjectComm((PetscObject)dm), &tr));
3455     PetscCall(DMPlexTransformSetDM(tr, dm));
3456     PetscCall(DMPlexTransformSetType(tr, DMPLEXEXTRUDE));
3457     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
3458     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)tr, prefix));
3459     PetscCall(PetscObjectGetOptions((PetscObject)dm, &options));
3460     PetscCall(PetscObjectSetOptions((PetscObject)tr, options));
3461     PetscCall(DMPlexTransformExtrudeSetLayers(tr, layers));
3462     PetscCall(DMPlexTransformExtrudeSetThickness(tr, thickness));
3463     PetscCall(DMPlexTransformExtrudeSetTensor(tr, PETSC_FALSE));
3464     PetscCall(DMPlexTransformExtrudeSetSymmetric(tr, PETSC_TRUE));
3465     PetscCall(DMPlexTransformExtrudeSetNormalFunction(tr, normalFunc));
3466     PetscCall(DMPlexTransformSetFromOptions(tr));
3467     PetscCall(PetscObjectSetOptions((PetscObject)tr, NULL));
3468     PetscCall(DMPlexTransformSetUp(tr));
3469     PetscCall(PetscObjectViewFromOptions((PetscObject)tr, NULL, "-dm_plex_tps_transform_view"));
3470     PetscCall(DMPlexTransformApply(tr, dm, &edm));
3471     PetscCall(DMCopyDisc(dm, edm));
3472     PetscCall(DMGetCoordinateDM(dm, &cdm));
3473     PetscCall(DMGetCoordinateDM(edm, &ecdm));
3474     PetscCall(DMCopyDisc(cdm, ecdm));
3475     PetscCall(DMPlexTransformCreateDiscLabels(tr, edm));
3476     PetscCall(DMPlexTransformDestroy(&tr));
3477     if (edm) {
3478       ((DM_Plex *)edm->data)->printFEM    = ((DM_Plex *)dm->data)->printFEM;
3479       ((DM_Plex *)edm->data)->printL2     = ((DM_Plex *)dm->data)->printL2;
3480       ((DM_Plex *)edm->data)->printLocate = ((DM_Plex *)dm->data)->printLocate;
3481     }
3482     PetscCall(DMPlexReplace_Internal(dm, &edm));
3483   }
3484   PetscFunctionReturn(0);
3485 }
3486 
3487 /*@
3488   DMPlexCreateTPSMesh - Create a distributed, interpolated mesh of a triply-periodic surface
3489 
3490   Collective
3491 
3492   Input Parameters:
3493 + comm   - The communicator for the `DM` object
3494 . tpstype - Type of triply-periodic surface
3495 . extent - Array of length 3 containing number of periods in each direction
3496 . periodic - array of length 3 with periodicity, or NULL for non-periodic
3497 . tps_distribute - Distribute 2D manifold mesh prior to refinement and extrusion (more scalable)
3498 . refinements - Number of factor-of-2 refinements of 2D manifold mesh
3499 . layers - Number of cell layers extruded in normal direction
3500 - thickness - Thickness in normal direction
3501 
3502   Output Parameter:
3503 . dm  - The `DM` object
3504 
3505   Level: beginner
3506 
3507   Notes:
3508   This meshes the surface of the Schwarz P or Gyroid surfaces.  Schwarz P is is the simplest member of the triply-periodic minimal surfaces.
3509   https://en.wikipedia.org/wiki/Schwarz_minimal_surface#Schwarz_P_(%22Primitive%22) and can be cut with "clean" boundaries.
3510   The Gyroid (https://en.wikipedia.org/wiki/Gyroid) is another triply-periodic minimal surface with applications in additive manufacturing; it is much more difficult to "cut" since there are no planes of symmetry.
3511   Our implementation creates a very coarse mesh of the surface and refines (by 4-way splitting) as many times as requested.
3512   On each refinement, all vertices are projected to their nearest point on the surface.
3513   This projection could readily be extended to related surfaces.
3514 
3515   The face (edge) sets for the Schwarz P surface are numbered 1(-x), 2(+x), 3(-y), 4(+y), 5(-z), 6(+z).
3516   When the mesh is refined, "Face Sets" contain the new vertices (created during refinement).  Use DMPlexLabelComplete() to propagate to coarse-level vertices.
3517 
3518   Developer Note:
3519   The Gyroid mesh does not currently mark boundary sets.
3520 
3521   References:
3522 . * - Maskery et al, Insights into the mechanical properties of several triply periodic minimal surface lattice structures made by polymer additive manufacturing, 2017.
3523     https://doi.org/10.1016/j.polymer.2017.11.049
3524 
3525 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMSetType()`, `DMCreate()`
3526 @*/
3527 PetscErrorCode DMPlexCreateTPSMesh(MPI_Comm comm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness, DM *dm)
3528 {
3529   PetscFunctionBegin;
3530   PetscCall(DMCreate(comm, dm));
3531   PetscCall(DMSetType(*dm, DMPLEX));
3532   PetscCall(DMPlexCreateTPSMesh_Internal(*dm, tpstype, extent, periodic, tps_distribute, refinements, layers, thickness));
3533   PetscFunctionReturn(0);
3534 }
3535 
3536 /*@
3537   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
3538 
3539   Collective
3540 
3541   Input Parameters:
3542 + comm    - The communicator for the `DM` object
3543 . dim     - The dimension
3544 . simplex - Use simplices, or tensor product cells
3545 - R       - The radius
3546 
3547   Output Parameter:
3548 . dm  - The `DM` object
3549 
3550   Level: beginner
3551 
3552 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBallMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3553 @*/
3554 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscReal R, DM *dm)
3555 {
3556   PetscFunctionBegin;
3557   PetscValidPointer(dm, 5);
3558   PetscCall(DMCreate(comm, dm));
3559   PetscCall(DMSetType(*dm, DMPLEX));
3560   PetscCall(DMPlexCreateSphereMesh_Internal(*dm, dim, simplex, R));
3561   PetscFunctionReturn(0);
3562 }
3563 
3564 static PetscErrorCode DMPlexCreateBallMesh_Internal(DM dm, PetscInt dim, PetscReal R)
3565 {
3566   DM      sdm, vol;
3567   DMLabel bdlabel;
3568 
3569   PetscFunctionBegin;
3570   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
3571   PetscCall(DMSetType(sdm, DMPLEX));
3572   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sdm, "bd_"));
3573   PetscCall(DMPlexCreateSphereMesh_Internal(sdm, dim - 1, PETSC_TRUE, R));
3574   PetscCall(DMSetFromOptions(sdm));
3575   PetscCall(DMViewFromOptions(sdm, NULL, "-dm_view"));
3576   PetscCall(DMPlexGenerate(sdm, NULL, PETSC_TRUE, &vol));
3577   PetscCall(DMDestroy(&sdm));
3578   PetscCall(DMPlexReplace_Internal(dm, &vol));
3579   PetscCall(DMCreateLabel(dm, "marker"));
3580   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
3581   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
3582   PetscCall(DMPlexLabelComplete(dm, bdlabel));
3583   PetscFunctionReturn(0);
3584 }
3585 
3586 /*@
3587   DMPlexCreateBallMesh - Creates a simplex mesh on the d-dimensional ball, B^d.
3588 
3589   Collective
3590 
3591   Input Parameters:
3592 + comm  - The communicator for the `DM` object
3593 . dim   - The dimension
3594 - R     - The radius
3595 
3596   Output Parameter:
3597 . dm  - The `DM` object
3598 
3599   Options Database Key:
3600 - bd_dm_refine - This will refine the surface mesh preserving the sphere geometry
3601 
3602   Level: beginner
3603 
3604 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3605 @*/
3606 PetscErrorCode DMPlexCreateBallMesh(MPI_Comm comm, PetscInt dim, PetscReal R, DM *dm)
3607 {
3608   PetscFunctionBegin;
3609   PetscCall(DMCreate(comm, dm));
3610   PetscCall(DMSetType(*dm, DMPLEX));
3611   PetscCall(DMPlexCreateBallMesh_Internal(*dm, dim, R));
3612   PetscFunctionReturn(0);
3613 }
3614 
3615 static PetscErrorCode DMPlexCreateReferenceCell_Internal(DM rdm, DMPolytopeType ct)
3616 {
3617   PetscFunctionBegin;
3618   switch (ct) {
3619   case DM_POLYTOPE_POINT: {
3620     PetscInt    numPoints[1]        = {1};
3621     PetscInt    coneSize[1]         = {0};
3622     PetscInt    cones[1]            = {0};
3623     PetscInt    coneOrientations[1] = {0};
3624     PetscScalar vertexCoords[1]     = {0.0};
3625 
3626     PetscCall(DMSetDimension(rdm, 0));
3627     PetscCall(DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3628   } break;
3629   case DM_POLYTOPE_SEGMENT: {
3630     PetscInt    numPoints[2]        = {2, 1};
3631     PetscInt    coneSize[3]         = {2, 0, 0};
3632     PetscInt    cones[2]            = {1, 2};
3633     PetscInt    coneOrientations[2] = {0, 0};
3634     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3635 
3636     PetscCall(DMSetDimension(rdm, 1));
3637     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3638   } break;
3639   case DM_POLYTOPE_POINT_PRISM_TENSOR: {
3640     PetscInt    numPoints[2]        = {2, 1};
3641     PetscInt    coneSize[3]         = {2, 0, 0};
3642     PetscInt    cones[2]            = {1, 2};
3643     PetscInt    coneOrientations[2] = {0, 0};
3644     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3645 
3646     PetscCall(DMSetDimension(rdm, 1));
3647     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3648   } break;
3649   case DM_POLYTOPE_TRIANGLE: {
3650     PetscInt    numPoints[2]        = {3, 1};
3651     PetscInt    coneSize[4]         = {3, 0, 0, 0};
3652     PetscInt    cones[3]            = {1, 2, 3};
3653     PetscInt    coneOrientations[3] = {0, 0, 0};
3654     PetscScalar vertexCoords[6]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0};
3655 
3656     PetscCall(DMSetDimension(rdm, 2));
3657     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3658   } break;
3659   case DM_POLYTOPE_QUADRILATERAL: {
3660     PetscInt    numPoints[2]        = {4, 1};
3661     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3662     PetscInt    cones[4]            = {1, 2, 3, 4};
3663     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3664     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0};
3665 
3666     PetscCall(DMSetDimension(rdm, 2));
3667     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3668   } break;
3669   case DM_POLYTOPE_SEG_PRISM_TENSOR: {
3670     PetscInt    numPoints[2]        = {4, 1};
3671     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3672     PetscInt    cones[4]            = {1, 2, 3, 4};
3673     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3674     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0};
3675 
3676     PetscCall(DMSetDimension(rdm, 2));
3677     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3678   } break;
3679   case DM_POLYTOPE_TETRAHEDRON: {
3680     PetscInt    numPoints[2]        = {4, 1};
3681     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3682     PetscInt    cones[4]            = {1, 2, 3, 4};
3683     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3684     PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0};
3685 
3686     PetscCall(DMSetDimension(rdm, 3));
3687     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3688   } break;
3689   case DM_POLYTOPE_HEXAHEDRON: {
3690     PetscInt    numPoints[2]        = {8, 1};
3691     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3692     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3693     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3694     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3695 
3696     PetscCall(DMSetDimension(rdm, 3));
3697     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3698   } break;
3699   case DM_POLYTOPE_TRI_PRISM: {
3700     PetscInt    numPoints[2]        = {6, 1};
3701     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3702     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3703     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3704     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3705 
3706     PetscCall(DMSetDimension(rdm, 3));
3707     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3708   } break;
3709   case DM_POLYTOPE_TRI_PRISM_TENSOR: {
3710     PetscInt    numPoints[2]        = {6, 1};
3711     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3712     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3713     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3714     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3715 
3716     PetscCall(DMSetDimension(rdm, 3));
3717     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3718   } break;
3719   case DM_POLYTOPE_QUAD_PRISM_TENSOR: {
3720     PetscInt    numPoints[2]        = {8, 1};
3721     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3722     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3723     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3724     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3725 
3726     PetscCall(DMSetDimension(rdm, 3));
3727     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3728   } break;
3729   case DM_POLYTOPE_PYRAMID: {
3730     PetscInt    numPoints[2]        = {5, 1};
3731     PetscInt    coneSize[6]         = {5, 0, 0, 0, 0, 0};
3732     PetscInt    cones[5]            = {1, 2, 3, 4, 5};
3733     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3734     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, 0.0, 0.0, 1.0};
3735 
3736     PetscCall(DMSetDimension(rdm, 3));
3737     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3738   } break;
3739   default:
3740     SETERRQ(PetscObjectComm((PetscObject)rdm), PETSC_ERR_ARG_WRONG, "Cannot create reference cell for cell type %s", DMPolytopeTypes[ct]);
3741   }
3742   {
3743     PetscInt Nv, v;
3744 
3745     /* Must create the celltype label here so that we do not automatically try to compute the types */
3746     PetscCall(DMCreateLabel(rdm, "celltype"));
3747     PetscCall(DMPlexSetCellType(rdm, 0, ct));
3748     PetscCall(DMPlexGetChart(rdm, NULL, &Nv));
3749     for (v = 1; v < Nv; ++v) PetscCall(DMPlexSetCellType(rdm, v, DM_POLYTOPE_POINT));
3750   }
3751   PetscCall(DMPlexInterpolateInPlace_Internal(rdm));
3752   PetscCall(PetscObjectSetName((PetscObject)rdm, DMPolytopeTypes[ct]));
3753   PetscFunctionReturn(0);
3754 }
3755 
3756 /*@
3757   DMPlexCreateReferenceCell - Create a `DMPLEX` with the appropriate FEM reference cell
3758 
3759   Collective
3760 
3761   Input Parameters:
3762 + comm - The communicator
3763 - ct   - The cell type of the reference cell
3764 
3765   Output Parameter:
3766 . refdm - The reference cell
3767 
3768   Level: intermediate
3769 
3770 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateReferenceCell()`, `DMPlexCreateBoxMesh()`
3771 @*/
3772 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, DMPolytopeType ct, DM *refdm)
3773 {
3774   PetscFunctionBegin;
3775   PetscCall(DMCreate(comm, refdm));
3776   PetscCall(DMSetType(*refdm, DMPLEX));
3777   PetscCall(DMPlexCreateReferenceCell_Internal(*refdm, ct));
3778   PetscFunctionReturn(0);
3779 }
3780 
3781 static PetscErrorCode DMPlexCreateBoundaryLabel_Private(DM dm, const char name[])
3782 {
3783   DM        plex;
3784   DMLabel   label;
3785   PetscBool hasLabel;
3786 
3787   PetscFunctionBegin;
3788   PetscCall(DMHasLabel(dm, name, &hasLabel));
3789   if (hasLabel) PetscFunctionReturn(0);
3790   PetscCall(DMCreateLabel(dm, name));
3791   PetscCall(DMGetLabel(dm, name, &label));
3792   PetscCall(DMConvert(dm, DMPLEX, &plex));
3793   PetscCall(DMPlexMarkBoundaryFaces(plex, 1, label));
3794   PetscCall(DMPlexLabelComplete(plex, label));
3795   PetscCall(DMDestroy(&plex));
3796   PetscFunctionReturn(0);
3797 }
3798 
3799 /*
3800   We use the last coordinate as the radius, the inner radius is lower[dim-1] and the outer radius is upper[dim-1]. Then we map the first coordinate around the circle.
3801 
3802     (x, y) -> (r, theta) = (x[1], (x[0] - lower[0]) * 2\pi/(upper[0] - lower[0]))
3803 */
3804 static void boxToAnnulus(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
3805 {
3806   const PetscReal low = PetscRealPart(constants[0]);
3807   const PetscReal upp = PetscRealPart(constants[1]);
3808   const PetscReal r   = PetscRealPart(u[1]);
3809   const PetscReal th  = 2. * PETSC_PI * (PetscRealPart(u[0]) - low) / (upp - low);
3810 
3811   f0[0] = r * PetscCosReal(th);
3812   f0[1] = r * PetscSinReal(th);
3813 }
3814 
3815 const char *const DMPlexShapes[] = {"box", "box_surface", "ball", "sphere", "cylinder", "schwarz_p", "gyroid", "doublet", "annulus", "hypercubic", "unknown", "DMPlexShape", "DM_SHAPE_", NULL};
3816 
3817 static PetscErrorCode DMPlexCreateFromOptions_Internal(PetscOptionItems *PetscOptionsObject, PetscBool *useCoordSpace, DM dm)
3818 {
3819   DMPlexShape    shape   = DM_SHAPE_BOX;
3820   DMPolytopeType cell    = DM_POLYTOPE_TRIANGLE;
3821   PetscInt       dim     = 2;
3822   PetscBool      simplex = PETSC_TRUE, interpolate = PETSC_TRUE, adjCone = PETSC_FALSE, adjClosure = PETSC_TRUE, refDomain = PETSC_FALSE;
3823   PetscBool      flg, flg2, fflg, bdfflg, nameflg;
3824   MPI_Comm       comm;
3825   char           filename[PETSC_MAX_PATH_LEN]   = "<unspecified>";
3826   char           bdFilename[PETSC_MAX_PATH_LEN] = "<unspecified>";
3827   char           plexname[PETSC_MAX_PATH_LEN]   = "";
3828 
3829   PetscFunctionBegin;
3830   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3831   /* TODO Turn this into a registration interface */
3832   PetscCall(PetscOptionsString("-dm_plex_filename", "File containing a mesh", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &fflg));
3833   PetscCall(PetscOptionsString("-dm_plex_boundary_filename", "File containing a mesh boundary", "DMPlexCreateFromFile", bdFilename, bdFilename, sizeof(bdFilename), &bdfflg));
3834   PetscCall(PetscOptionsString("-dm_plex_name", "Name of the mesh in the file", "DMPlexCreateFromFile", plexname, plexname, sizeof(plexname), &nameflg));
3835   PetscCall(PetscOptionsEnum("-dm_plex_cell", "Cell shape", "", DMPolytopeTypes, (PetscEnum)cell, (PetscEnum *)&cell, NULL));
3836   PetscCall(PetscOptionsBool("-dm_plex_reference_cell_domain", "Use a reference cell domain", "", refDomain, &refDomain, NULL));
3837   PetscCall(PetscOptionsEnum("-dm_plex_shape", "Shape for built-in mesh", "", DMPlexShapes, (PetscEnum)shape, (PetscEnum *)&shape, &flg));
3838   PetscCall(PetscOptionsBoundedInt("-dm_plex_dim", "Topological dimension of the mesh", "DMGetDimension", dim, &dim, &flg, 0));
3839   PetscCheck(dim >= 0, comm, PETSC_ERR_ARG_OUTOFRANGE, "Dimension %" PetscInt_FMT " should be in [0, infinity)", dim);
3840   PetscCall(PetscOptionsBool("-dm_plex_simplex", "Mesh cell shape", "", simplex, &simplex, &flg));
3841   PetscCall(PetscOptionsBool("-dm_plex_interpolate", "Flag to create edges and faces automatically", "", interpolate, &interpolate, &flg));
3842   PetscCall(PetscOptionsBool("-dm_plex_adj_cone", "Set adjacency direction", "DMSetBasicAdjacency", adjCone, &adjCone, &flg));
3843   PetscCall(PetscOptionsBool("-dm_plex_adj_closure", "Set adjacency size", "DMSetBasicAdjacency", adjClosure, &adjClosure, &flg2));
3844   if (flg || flg2) PetscCall(DMSetBasicAdjacency(dm, adjCone, adjClosure));
3845 
3846   switch (cell) {
3847   case DM_POLYTOPE_POINT:
3848   case DM_POLYTOPE_SEGMENT:
3849   case DM_POLYTOPE_POINT_PRISM_TENSOR:
3850   case DM_POLYTOPE_TRIANGLE:
3851   case DM_POLYTOPE_QUADRILATERAL:
3852   case DM_POLYTOPE_TETRAHEDRON:
3853   case DM_POLYTOPE_HEXAHEDRON:
3854     *useCoordSpace = PETSC_TRUE;
3855     break;
3856   default:
3857     *useCoordSpace = PETSC_FALSE;
3858     break;
3859   }
3860 
3861   if (fflg) {
3862     DM dmnew;
3863 
3864     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), filename, plexname, interpolate, &dmnew));
3865     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3866     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3867   } else if (refDomain) {
3868     PetscCall(DMPlexCreateReferenceCell_Internal(dm, cell));
3869   } else if (bdfflg) {
3870     DM bdm, dmnew;
3871 
3872     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), bdFilename, plexname, interpolate, &bdm));
3873     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)bdm, "bd_"));
3874     PetscCall(DMSetFromOptions(bdm));
3875     PetscCall(DMPlexGenerate(bdm, NULL, interpolate, &dmnew));
3876     PetscCall(DMDestroy(&bdm));
3877     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3878     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3879   } else {
3880     PetscCall(PetscObjectSetName((PetscObject)dm, DMPlexShapes[shape]));
3881     switch (shape) {
3882     case DM_SHAPE_BOX:
3883     case DM_SHAPE_ANNULUS: {
3884       PetscInt       faces[3]  = {0, 0, 0};
3885       PetscReal      lower[3]  = {0, 0, 0};
3886       PetscReal      upper[3]  = {1, 1, 1};
3887       DMBoundaryType bdt[3]    = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3888       PetscBool      isAnnular = shape == DM_SHAPE_ANNULUS ? PETSC_TRUE : PETSC_FALSE;
3889       PetscInt       i, n;
3890 
3891       n = dim;
3892       for (i = 0; i < dim; ++i) faces[i] = (dim == 1 ? 1 : 4 - dim);
3893       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3894       n = 3;
3895       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3896       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3897       n = 3;
3898       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3899       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3900       n = 3;
3901       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
3902       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3903 
3904       PetscCheck(!isAnnular || dim == 2, comm, PETSC_ERR_ARG_OUTOFRANGE, "Only two dimensional annuli have been implemented");
3905       if (isAnnular)
3906         for (i = 0; i < dim - 1; ++i) bdt[i] = DM_BOUNDARY_PERIODIC;
3907 
3908       switch (cell) {
3909       case DM_POLYTOPE_TRI_PRISM_TENSOR:
3910         PetscCall(DMPlexCreateWedgeBoxMesh_Internal(dm, faces, lower, upper, bdt));
3911         if (!interpolate) {
3912           DM udm;
3913 
3914           PetscCall(DMPlexUninterpolate(dm, &udm));
3915           PetscCall(DMPlexReplace_Internal(dm, &udm));
3916         }
3917         break;
3918       default:
3919         PetscCall(DMPlexCreateBoxMesh_Internal(dm, dim, simplex, faces, lower, upper, bdt, interpolate));
3920         break;
3921       }
3922       if (isAnnular) {
3923         DM          cdm;
3924         PetscDS     cds;
3925         PetscScalar bounds[2] = {lower[0], upper[0]};
3926 
3927         // Fix coordinates for annular region
3928         PetscCall(DMSetPeriodicity(dm, NULL, NULL, NULL));
3929         PetscCall(DMSetCellCoordinatesLocal(dm, NULL));
3930         PetscCall(DMSetCellCoordinates(dm, NULL));
3931         PetscCall(DMPlexCreateCoordinateSpace(dm, 1, NULL));
3932         PetscCall(DMGetCoordinateDM(dm, &cdm));
3933         PetscCall(DMGetDS(cdm, &cds));
3934         PetscCall(PetscDSSetConstants(cds, 2, bounds));
3935         PetscCall(DMPlexRemapGeometry(dm, 0.0, boxToAnnulus));
3936       }
3937     } break;
3938     case DM_SHAPE_BOX_SURFACE: {
3939       PetscInt  faces[3] = {0, 0, 0};
3940       PetscReal lower[3] = {0, 0, 0};
3941       PetscReal upper[3] = {1, 1, 1};
3942       PetscInt  i, n;
3943 
3944       n = dim + 1;
3945       for (i = 0; i < dim + 1; ++i) faces[i] = (dim + 1 == 1 ? 1 : 4 - (dim + 1));
3946       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3947       n = 3;
3948       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3949       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3950       n = 3;
3951       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3952       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3953       PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(dm, dim + 1, faces, lower, upper, interpolate));
3954     } break;
3955     case DM_SHAPE_SPHERE: {
3956       PetscReal R = 1.0;
3957 
3958       PetscCall(PetscOptionsReal("-dm_plex_sphere_radius", "Radius of the sphere", "", R, &R, &flg));
3959       PetscCall(DMPlexCreateSphereMesh_Internal(dm, dim, simplex, R));
3960     } break;
3961     case DM_SHAPE_BALL: {
3962       PetscReal R = 1.0;
3963 
3964       PetscCall(PetscOptionsReal("-dm_plex_ball_radius", "Radius of the ball", "", R, &R, &flg));
3965       PetscCall(DMPlexCreateBallMesh_Internal(dm, dim, R));
3966     } break;
3967     case DM_SHAPE_CYLINDER: {
3968       DMBoundaryType bdt = DM_BOUNDARY_NONE;
3969       PetscInt       Nw  = 6;
3970 
3971       PetscCall(PetscOptionsEnum("-dm_plex_cylinder_bd", "Boundary type in the z direction", "", DMBoundaryTypes, (PetscEnum)bdt, (PetscEnum *)&bdt, NULL));
3972       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_wedges", "Number of wedges around the cylinder", "", Nw, &Nw, NULL));
3973       switch (cell) {
3974       case DM_POLYTOPE_TRI_PRISM_TENSOR:
3975         PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(dm, Nw, interpolate));
3976         break;
3977       default:
3978         PetscCall(DMPlexCreateHexCylinderMesh_Internal(dm, bdt));
3979         break;
3980       }
3981     } break;
3982     case DM_SHAPE_SCHWARZ_P: // fallthrough
3983     case DM_SHAPE_GYROID: {
3984       PetscInt       extent[3] = {1, 1, 1}, refine = 0, layers = 0, three;
3985       PetscReal      thickness   = 0.;
3986       DMBoundaryType periodic[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3987       DMPlexTPSType  tps_type    = shape == DM_SHAPE_SCHWARZ_P ? DMPLEX_TPS_SCHWARZ_P : DMPLEX_TPS_GYROID;
3988       PetscBool      tps_distribute;
3989       PetscCall(PetscOptionsIntArray("-dm_plex_tps_extent", "Number of replicas for each of three dimensions", NULL, extent, (three = 3, &three), NULL));
3990       PetscCall(PetscOptionsInt("-dm_plex_tps_refine", "Number of refinements", NULL, refine, &refine, NULL));
3991       PetscCall(PetscOptionsEnumArray("-dm_plex_tps_periodic", "Periodicity in each of three dimensions", NULL, DMBoundaryTypes, (PetscEnum *)periodic, (three = 3, &three), NULL));
3992       PetscCall(PetscOptionsInt("-dm_plex_tps_layers", "Number of layers in volumetric extrusion (or zero to not extrude)", NULL, layers, &layers, NULL));
3993       PetscCall(PetscOptionsReal("-dm_plex_tps_thickness", "Thickness of volumetric extrusion", NULL, thickness, &thickness, NULL));
3994       PetscCall(DMPlexDistributeGetDefault(dm, &tps_distribute));
3995       PetscCall(PetscOptionsBool("-dm_plex_tps_distribute", "Distribute the 2D mesh prior to refinement and extrusion", NULL, tps_distribute, &tps_distribute, NULL));
3996       PetscCall(DMPlexCreateTPSMesh_Internal(dm, tps_type, extent, periodic, tps_distribute, refine, layers, thickness));
3997     } break;
3998     case DM_SHAPE_DOUBLET: {
3999       DM        dmnew;
4000       PetscReal rl = 0.0;
4001 
4002       PetscCall(PetscOptionsReal("-dm_plex_doublet_refinementlimit", "Refinement limit", NULL, rl, &rl, NULL));
4003       PetscCall(DMPlexCreateDoublet(PetscObjectComm((PetscObject)dm), dim, simplex, interpolate, rl, &dmnew));
4004       PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4005       PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4006     } break;
4007     case DM_SHAPE_HYPERCUBIC: {
4008       PetscInt       *edges;
4009       PetscReal      *lower, *upper;
4010       DMBoundaryType *bdt;
4011       PetscInt        n, d;
4012 
4013       *useCoordSpace = PETSC_FALSE;
4014       PetscCall(PetscMalloc4(dim, &edges, dim, &lower, dim, &upper, dim, &bdt));
4015       for (d = 0; d < dim; ++d) {
4016         edges[d] = 1;
4017         lower[d] = 0.;
4018         upper[d] = 1.;
4019         bdt[d]   = DM_BOUNDARY_PERIODIC;
4020       }
4021       n = dim;
4022       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", edges, &n, &flg));
4023       n = dim;
4024       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
4025       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4026       n = dim;
4027       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
4028       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4029       n = dim;
4030       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
4031       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4032       PetscCall(DMPlexCreateHypercubicMesh_Internal(dm, dim, lower, upper, edges, bdt));
4033       PetscCall(PetscFree4(edges, lower, upper, bdt));
4034     } break;
4035     default:
4036       SETERRQ(comm, PETSC_ERR_SUP, "Domain shape %s is unsupported", DMPlexShapes[shape]);
4037     }
4038   }
4039   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
4040   if (!((PetscObject)dm)->name && nameflg) PetscCall(PetscObjectSetName((PetscObject)dm, plexname));
4041   PetscFunctionReturn(0);
4042 }
4043 
4044 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
4045 {
4046   DM_Plex  *mesh = (DM_Plex *)dm->data;
4047   PetscBool flg, flg2;
4048   char      bdLabel[PETSC_MAX_PATH_LEN];
4049 
4050   PetscFunctionBegin;
4051   /* Handle viewing */
4052   PetscCall(PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL));
4053   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fem", "Debug output level all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL, 0));
4054   PetscCall(PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL));
4055   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL, 0));
4056   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_locate", "Debug output level all point location computations", "DMLocatePoints", 0, &mesh->printLocate, NULL, 0));
4057   PetscCall(DMMonitorSetFromOptions(dm, "-dm_plex_monitor_throughput", "Monitor the simulation throughput", "DMPlexMonitorThroughput", DMPlexMonitorThroughput, NULL, &flg));
4058   if (flg) PetscCall(PetscLogDefaultBegin());
4059   /* Labeling */
4060   PetscCall(PetscOptionsString("-dm_plex_boundary_label", "Label to mark the mesh boundary", "", bdLabel, bdLabel, sizeof(bdLabel), &flg));
4061   if (flg) PetscCall(DMPlexCreateBoundaryLabel_Private(dm, bdLabel));
4062   /* Point Location */
4063   PetscCall(PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL));
4064   /* Partitioning and distribution */
4065   PetscCall(PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL));
4066   /* Generation and remeshing */
4067   PetscCall(PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL));
4068   /* Projection behavior */
4069   PetscCall(PetscOptionsBoundedInt("-dm_plex_max_projection_height", "Maximum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL, 0));
4070   PetscCall(PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL));
4071   /* Checking structure */
4072   {
4073     PetscBool all = PETSC_FALSE;
4074 
4075     PetscCall(PetscOptionsBool("-dm_plex_check_all", "Perform all basic checks", "DMPlexCheck", PETSC_FALSE, &all, NULL));
4076     if (all) {
4077       PetscCall(DMPlexCheck(dm));
4078     } else {
4079       PetscCall(PetscOptionsBool("-dm_plex_check_symmetry", "Check that the adjacency information in the mesh is symmetric", "DMPlexCheckSymmetry", PETSC_FALSE, &flg, &flg2));
4080       if (flg && flg2) PetscCall(DMPlexCheckSymmetry(dm));
4081       PetscCall(PetscOptionsBool("-dm_plex_check_skeleton", "Check that each cell has the correct number of vertices (only for homogeneous simplex or tensor meshes)", "DMPlexCheckSkeleton", PETSC_FALSE, &flg, &flg2));
4082       if (flg && flg2) PetscCall(DMPlexCheckSkeleton(dm, 0));
4083       PetscCall(PetscOptionsBool("-dm_plex_check_faces", "Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type", "DMPlexCheckFaces", PETSC_FALSE, &flg, &flg2));
4084       if (flg && flg2) PetscCall(DMPlexCheckFaces(dm, 0));
4085       PetscCall(PetscOptionsBool("-dm_plex_check_geometry", "Check that cells have positive volume", "DMPlexCheckGeometry", PETSC_FALSE, &flg, &flg2));
4086       if (flg && flg2) PetscCall(DMPlexCheckGeometry(dm));
4087       PetscCall(PetscOptionsBool("-dm_plex_check_pointsf", "Check some necessary conditions for PointSF", "DMPlexCheckPointSF", PETSC_FALSE, &flg, &flg2));
4088       if (flg && flg2) PetscCall(DMPlexCheckPointSF(dm, NULL, PETSC_FALSE));
4089       PetscCall(PetscOptionsBool("-dm_plex_check_interface_cones", "Check points on inter-partition interfaces have conforming order of cone points", "DMPlexCheckInterfaceCones", PETSC_FALSE, &flg, &flg2));
4090       if (flg && flg2) PetscCall(DMPlexCheckInterfaceCones(dm));
4091     }
4092     PetscCall(PetscOptionsBool("-dm_plex_check_cell_shape", "Check cell shape", "DMPlexCheckCellShape", PETSC_FALSE, &flg, &flg2));
4093     if (flg && flg2) PetscCall(DMPlexCheckCellShape(dm, PETSC_TRUE, PETSC_DETERMINE));
4094   }
4095   {
4096     PetscReal scale = 1.0;
4097 
4098     PetscCall(PetscOptionsReal("-dm_plex_scale", "Scale factor for mesh coordinates", "DMPlexScale", scale, &scale, &flg));
4099     if (flg) {
4100       Vec coordinates, coordinatesLocal;
4101 
4102       PetscCall(DMGetCoordinates(dm, &coordinates));
4103       PetscCall(DMGetCoordinatesLocal(dm, &coordinatesLocal));
4104       PetscCall(VecScale(coordinates, scale));
4105       PetscCall(VecScale(coordinatesLocal, scale));
4106     }
4107   }
4108   PetscCall(PetscPartitionerSetFromOptions(mesh->partitioner));
4109   PetscFunctionReturn(0);
4110 }
4111 
4112 PetscErrorCode DMSetFromOptions_Overlap_Plex(DM dm, PetscOptionItems *PetscOptionsObject, PetscInt *overlap)
4113 {
4114   PetscInt  numOvLabels = 16, numOvExLabels = 16;
4115   char     *ovLabelNames[16], *ovExLabelNames[16];
4116   PetscInt  numOvValues = 16, numOvExValues = 16, l;
4117   PetscBool flg;
4118 
4119   PetscFunctionBegin;
4120   PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", *overlap, overlap, NULL, 0));
4121   PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_labels", "List of overlap label names", "DMPlexDistribute", ovLabelNames, &numOvLabels, &flg));
4122   if (!flg) numOvLabels = 0;
4123   if (numOvLabels) {
4124     ((DM_Plex *)dm->data)->numOvLabels = numOvLabels;
4125     for (l = 0; l < numOvLabels; ++l) {
4126       PetscCall(DMGetLabel(dm, ovLabelNames[l], &((DM_Plex *)dm->data)->ovLabels[l]));
4127       PetscCheck(((DM_Plex *)dm->data)->ovLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovLabelNames[l]);
4128       PetscCall(PetscFree(ovLabelNames[l]));
4129     }
4130     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_values", "List of overlap label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovValues, &numOvValues, &flg));
4131     if (!flg) numOvValues = 0;
4132     PetscCheck(numOvLabels == numOvValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvLabels, numOvValues);
4133 
4134     PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_exclude_labels", "List of overlap exclude label names", "DMPlexDistribute", ovExLabelNames, &numOvExLabels, &flg));
4135     if (!flg) numOvExLabels = 0;
4136     ((DM_Plex *)dm->data)->numOvExLabels = numOvExLabels;
4137     for (l = 0; l < numOvExLabels; ++l) {
4138       PetscCall(DMGetLabel(dm, ovExLabelNames[l], &((DM_Plex *)dm->data)->ovExLabels[l]));
4139       PetscCheck(((DM_Plex *)dm->data)->ovExLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovExLabelNames[l]);
4140       PetscCall(PetscFree(ovExLabelNames[l]));
4141     }
4142     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_exclude_values", "List of overlap exclude label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovExValues, &numOvExValues, &flg));
4143     if (!flg) numOvExValues = 0;
4144     PetscCheck(numOvExLabels == numOvExValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of exclude labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvExLabels, numOvExValues);
4145   }
4146   PetscFunctionReturn(0);
4147 }
4148 
4149 static PetscErrorCode DMSetFromOptions_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
4150 {
4151   PetscFunctionList        ordlist;
4152   char                     oname[256];
4153   PetscReal                volume    = -1.0;
4154   PetscInt                 prerefine = 0, refine = 0, r, coarsen = 0, overlap = 0, extLayers = 0, dim;
4155   PetscBool                uniformOrig, created = PETSC_FALSE, uniform = PETSC_TRUE, distribute, interpolate = PETSC_TRUE, coordSpace = PETSC_TRUE, remap = PETSC_TRUE, ghostCells = PETSC_FALSE, isHierarchy, ignoreModel = PETSC_FALSE, flg;
4156   DMPlexReorderDefaultFlag reorder;
4157 
4158   PetscFunctionBegin;
4159   PetscOptionsHeadBegin(PetscOptionsObject, "DMPlex Options");
4160   if (dm->cloneOpts) goto non_refine;
4161   /* Handle automatic creation */
4162   PetscCall(DMGetDimension(dm, &dim));
4163   if (dim < 0) {
4164     PetscCall(DMPlexCreateFromOptions_Internal(PetscOptionsObject, &coordSpace, dm));
4165     created = PETSC_TRUE;
4166   }
4167   PetscCall(DMGetDimension(dm, &dim));
4168   /* Handle interpolation before distribution */
4169   PetscCall(PetscOptionsBool("-dm_plex_interpolate_pre", "Flag to interpolate mesh before distribution", "", interpolate, &interpolate, &flg));
4170   if (flg) {
4171     DMPlexInterpolatedFlag interpolated;
4172 
4173     PetscCall(DMPlexIsInterpolated(dm, &interpolated));
4174     if (interpolated == DMPLEX_INTERPOLATED_FULL && !interpolate) {
4175       DM udm;
4176 
4177       PetscCall(DMPlexUninterpolate(dm, &udm));
4178       PetscCall(DMPlexReplace_Internal(dm, &udm));
4179     } else if (interpolated != DMPLEX_INTERPOLATED_FULL && interpolate) {
4180       DM idm;
4181 
4182       PetscCall(DMPlexInterpolate(dm, &idm));
4183       PetscCall(DMPlexReplace_Internal(dm, &idm));
4184     }
4185   }
4186   /* Handle DMPlex refinement before distribution */
4187   PetscCall(PetscOptionsBool("-dm_refine_ignore_model", "Flag to ignore the geometry model when refining", "DMCreate", ignoreModel, &ignoreModel, &flg));
4188   if (flg) ((DM_Plex *)dm->data)->ignoreModel = ignoreModel;
4189   PetscCall(DMPlexGetRefinementUniform(dm, &uniformOrig));
4190   PetscCall(PetscOptionsBoundedInt("-dm_refine_pre", "The number of refinements before distribution", "DMCreate", prerefine, &prerefine, NULL, 0));
4191   PetscCall(PetscOptionsBool("-dm_refine_remap_pre", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
4192   PetscCall(PetscOptionsBool("-dm_refine_uniform_pre", "Flag for uniform refinement before distribution", "DMCreate", uniform, &uniform, &flg));
4193   if (flg) PetscCall(DMPlexSetRefinementUniform(dm, uniform));
4194   PetscCall(PetscOptionsReal("-dm_refine_volume_limit_pre", "The maximum cell volume after refinement before distribution", "DMCreate", volume, &volume, &flg));
4195   if (flg) {
4196     PetscCall(DMPlexSetRefinementUniform(dm, PETSC_FALSE));
4197     PetscCall(DMPlexSetRefinementLimit(dm, volume));
4198     prerefine = PetscMax(prerefine, 1);
4199   }
4200   for (r = 0; r < prerefine; ++r) {
4201     DM             rdm;
4202     PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4203 
4204     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4205     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
4206     PetscCall(DMPlexReplace_Internal(dm, &rdm));
4207     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4208     if (coordFunc && remap) {
4209       PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4210       ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4211     }
4212   }
4213   PetscCall(DMPlexSetRefinementUniform(dm, uniformOrig));
4214   /* Handle DMPlex extrusion before distribution */
4215   PetscCall(PetscOptionsBoundedInt("-dm_extrude", "The number of layers to extrude", "", extLayers, &extLayers, NULL, 0));
4216   if (extLayers) {
4217     DM edm;
4218 
4219     PetscCall(DMExtrude(dm, extLayers, &edm));
4220     PetscCall(DMPlexReplace_Internal(dm, &edm));
4221     ((DM_Plex *)dm->data)->coordFunc = NULL;
4222     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4223     extLayers = 0;
4224     PetscCall(DMGetDimension(dm, &dim));
4225   }
4226   /* Handle DMPlex reordering before distribution */
4227   PetscCall(DMPlexReorderGetDefault(dm, &reorder));
4228   PetscCall(MatGetOrderingList(&ordlist));
4229   PetscCall(PetscStrncpy(oname, MATORDERINGNATURAL, sizeof(oname)));
4230   PetscCall(PetscOptionsFList("-dm_plex_reorder", "Set mesh reordering type", "DMPlexGetOrdering", ordlist, MATORDERINGNATURAL, oname, sizeof(oname), &flg));
4231   if (reorder == DMPLEX_REORDER_DEFAULT_TRUE || flg) {
4232     DM pdm;
4233     IS perm;
4234 
4235     PetscCall(DMPlexGetOrdering(dm, oname, NULL, &perm));
4236     PetscCall(DMPlexPermute(dm, perm, &pdm));
4237     PetscCall(ISDestroy(&perm));
4238     PetscCall(DMPlexReplace_Internal(dm, &pdm));
4239     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4240   }
4241   /* Handle DMPlex distribution */
4242   PetscCall(DMPlexDistributeGetDefault(dm, &distribute));
4243   PetscCall(PetscOptionsBool("-dm_distribute", "Flag to redistribute a mesh among processes", "DMPlexDistribute", distribute, &distribute, NULL));
4244   PetscCall(DMSetFromOptions_Overlap_Plex(dm, PetscOptionsObject, &overlap));
4245   if (distribute) {
4246     DM               pdm = NULL;
4247     PetscPartitioner part;
4248 
4249     PetscCall(DMPlexGetPartitioner(dm, &part));
4250     PetscCall(PetscPartitionerSetFromOptions(part));
4251     PetscCall(DMPlexDistribute(dm, overlap, NULL, &pdm));
4252     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
4253   }
4254   /* Create coordinate space */
4255   if (created) {
4256     DM_Plex  *mesh   = (DM_Plex *)dm->data;
4257     PetscInt  degree = 1;
4258     PetscInt  height = 0;
4259     DM        cdm;
4260     PetscBool flg;
4261 
4262     PetscCall(PetscOptionsBool("-dm_coord_space", "Use an FEM space for coordinates", "", coordSpace, &coordSpace, &flg));
4263     PetscCall(PetscOptionsInt("-dm_coord_petscspace_degree", "FEM degree for coordinate space", "", degree, &degree, NULL));
4264     if (coordSpace) PetscCall(DMPlexCreateCoordinateSpace(dm, degree, mesh->coordFunc));
4265     PetscCall(DMGetCoordinateDM(dm, &cdm));
4266     if (flg && !coordSpace) {
4267       PetscDS      cds;
4268       PetscObject  obj;
4269       PetscClassId id;
4270 
4271       PetscCall(DMGetDS(cdm, &cds));
4272       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
4273       PetscCall(PetscObjectGetClassId(obj, &id));
4274       if (id == PETSCFE_CLASSID) {
4275         PetscContainer dummy;
4276 
4277         PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &dummy));
4278         PetscCall(PetscObjectSetName((PetscObject)dummy, "coordinates"));
4279         PetscCall(DMSetField(cdm, 0, NULL, (PetscObject)dummy));
4280         PetscCall(PetscContainerDestroy(&dummy));
4281         PetscCall(DMClearDS(cdm));
4282       }
4283       mesh->coordFunc = NULL;
4284     }
4285     PetscCall(PetscOptionsBool("-dm_sparse_localize", "Localize only necessary cells", "", dm->sparseLocalize, &dm->sparseLocalize, &flg));
4286     PetscCall(PetscOptionsInt("-dm_localize_height", "Localize edges and faces in addition to cells", "", height, &height, &flg));
4287     if (flg) PetscCall(DMPlexSetMaxProjectionHeight(cdm, height));
4288     PetscCall(DMLocalizeCoordinates(dm));
4289   }
4290   /* Handle DMPlex refinement */
4291   remap = PETSC_TRUE;
4292   PetscCall(PetscOptionsBoundedInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL, 0));
4293   PetscCall(PetscOptionsBool("-dm_refine_remap", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
4294   PetscCall(PetscOptionsBoundedInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy, 0));
4295   if (refine) PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
4296   if (refine && isHierarchy) {
4297     DM *dms, coarseDM;
4298 
4299     PetscCall(DMGetCoarseDM(dm, &coarseDM));
4300     PetscCall(PetscObjectReference((PetscObject)coarseDM));
4301     PetscCall(PetscMalloc1(refine, &dms));
4302     PetscCall(DMRefineHierarchy(dm, refine, dms));
4303     /* Total hack since we do not pass in a pointer */
4304     PetscCall(DMPlexSwap_Static(dm, dms[refine - 1]));
4305     if (refine == 1) {
4306       PetscCall(DMSetCoarseDM(dm, dms[0]));
4307       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4308     } else {
4309       PetscCall(DMSetCoarseDM(dm, dms[refine - 2]));
4310       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4311       PetscCall(DMSetCoarseDM(dms[0], dms[refine - 1]));
4312       PetscCall(DMPlexSetRegularRefinement(dms[0], PETSC_TRUE));
4313     }
4314     PetscCall(DMSetCoarseDM(dms[refine - 1], coarseDM));
4315     PetscCall(PetscObjectDereference((PetscObject)coarseDM));
4316     /* Free DMs */
4317     for (r = 0; r < refine; ++r) {
4318       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4319       PetscCall(DMDestroy(&dms[r]));
4320     }
4321     PetscCall(PetscFree(dms));
4322   } else {
4323     for (r = 0; r < refine; ++r) {
4324       DM             rdm;
4325       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4326 
4327       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4328       PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
4329       /* Total hack since we do not pass in a pointer */
4330       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4331       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4332       if (coordFunc && remap) {
4333         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4334         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4335       }
4336     }
4337   }
4338   /* Handle DMPlex coarsening */
4339   PetscCall(PetscOptionsBoundedInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL, 0));
4340   PetscCall(PetscOptionsBoundedInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy, 0));
4341   if (coarsen && isHierarchy) {
4342     DM *dms;
4343 
4344     PetscCall(PetscMalloc1(coarsen, &dms));
4345     PetscCall(DMCoarsenHierarchy(dm, coarsen, dms));
4346     /* Free DMs */
4347     for (r = 0; r < coarsen; ++r) {
4348       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4349       PetscCall(DMDestroy(&dms[r]));
4350     }
4351     PetscCall(PetscFree(dms));
4352   } else {
4353     for (r = 0; r < coarsen; ++r) {
4354       DM             cdm;
4355       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4356 
4357       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4358       PetscCall(DMCoarsen(dm, PetscObjectComm((PetscObject)dm), &cdm));
4359       /* Total hack since we do not pass in a pointer */
4360       PetscCall(DMPlexReplace_Internal(dm, &cdm));
4361       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4362       if (coordFunc) {
4363         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4364         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4365       }
4366     }
4367   }
4368   /* Handle ghost cells */
4369   PetscCall(PetscOptionsBool("-dm_plex_create_fv_ghost_cells", "Flag to create finite volume ghost cells on the boundary", "DMCreate", ghostCells, &ghostCells, NULL));
4370   if (ghostCells) {
4371     DM   gdm;
4372     char lname[PETSC_MAX_PATH_LEN];
4373 
4374     lname[0] = '\0';
4375     PetscCall(PetscOptionsString("-dm_plex_fv_ghost_cells_label", "Label name for ghost cells boundary", "DMCreate", lname, lname, sizeof(lname), &flg));
4376     PetscCall(DMPlexConstructGhostCells(dm, flg ? lname : NULL, NULL, &gdm));
4377     PetscCall(DMPlexReplace_Internal(dm, &gdm));
4378   }
4379   /* Handle 1D order */
4380   if (reorder != DMPLEX_REORDER_DEFAULT_FALSE && dim == 1) {
4381     DM           cdm, rdm;
4382     PetscDS      cds;
4383     PetscObject  obj;
4384     PetscClassId id = PETSC_OBJECT_CLASSID;
4385     IS           perm;
4386     PetscInt     Nf;
4387     PetscBool    distributed;
4388 
4389     PetscCall(DMPlexIsDistributed(dm, &distributed));
4390     PetscCall(DMGetCoordinateDM(dm, &cdm));
4391     PetscCall(DMGetDS(cdm, &cds));
4392     PetscCall(PetscDSGetNumFields(cds, &Nf));
4393     if (Nf) {
4394       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
4395       PetscCall(PetscObjectGetClassId(obj, &id));
4396     }
4397     if (!distributed && id != PETSCFE_CLASSID) {
4398       PetscCall(DMPlexGetOrdering1D(dm, &perm));
4399       PetscCall(DMPlexPermute(dm, perm, &rdm));
4400       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4401       PetscCall(ISDestroy(&perm));
4402     }
4403   }
4404 /* Handle */
4405 non_refine:
4406   PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4407   PetscOptionsHeadEnd();
4408   PetscFunctionReturn(0);
4409 }
4410 
4411 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm, Vec *vec)
4412 {
4413   PetscFunctionBegin;
4414   PetscCall(DMCreateGlobalVector_Section_Private(dm, vec));
4415   /* PetscCall(VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM)); */
4416   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex));
4417   PetscCall(VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void))VecView_Plex_Native));
4418   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex));
4419   PetscCall(VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void))VecLoad_Plex_Native));
4420   PetscFunctionReturn(0);
4421 }
4422 
4423 static PetscErrorCode DMCreateLocalVector_Plex(DM dm, Vec *vec)
4424 {
4425   PetscFunctionBegin;
4426   PetscCall(DMCreateLocalVector_Section_Private(dm, vec));
4427   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex_Local));
4428   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex_Local));
4429   PetscFunctionReturn(0);
4430 }
4431 
4432 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
4433 {
4434   PetscInt depth, d;
4435 
4436   PetscFunctionBegin;
4437   PetscCall(DMPlexGetDepth(dm, &depth));
4438   if (depth == 1) {
4439     PetscCall(DMGetDimension(dm, &d));
4440     if (dim == 0) PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4441     else if (dim == d) PetscCall(DMPlexGetDepthStratum(dm, 1, pStart, pEnd));
4442     else {
4443       *pStart = 0;
4444       *pEnd   = 0;
4445     }
4446   } else {
4447     PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4448   }
4449   PetscFunctionReturn(0);
4450 }
4451 
4452 static PetscErrorCode DMGetNeighbors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
4453 {
4454   PetscSF            sf;
4455   PetscInt           niranks, njranks, n;
4456   const PetscMPIInt *iranks, *jranks;
4457   DM_Plex           *data = (DM_Plex *)dm->data;
4458 
4459   PetscFunctionBegin;
4460   PetscCall(DMGetPointSF(dm, &sf));
4461   if (!data->neighbors) {
4462     PetscCall(PetscSFSetUp(sf));
4463     PetscCall(PetscSFGetRootRanks(sf, &njranks, &jranks, NULL, NULL, NULL));
4464     PetscCall(PetscSFGetLeafRanks(sf, &niranks, &iranks, NULL, NULL));
4465     PetscCall(PetscMalloc1(njranks + niranks + 1, &data->neighbors));
4466     PetscCall(PetscArraycpy(data->neighbors + 1, jranks, njranks));
4467     PetscCall(PetscArraycpy(data->neighbors + njranks + 1, iranks, niranks));
4468     n = njranks + niranks;
4469     PetscCall(PetscSortRemoveDupsMPIInt(&n, data->neighbors + 1));
4470     /* The following cast should never fail: can't have more neighbors than PETSC_MPI_INT_MAX */
4471     PetscCall(PetscMPIIntCast(n, data->neighbors));
4472   }
4473   if (nranks) *nranks = data->neighbors[0];
4474   if (ranks) {
4475     if (data->neighbors[0]) *ranks = data->neighbors + 1;
4476     else *ranks = NULL;
4477   }
4478   PetscFunctionReturn(0);
4479 }
4480 
4481 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM, DM, Mat, Vec, Vec);
4482 
4483 static PetscErrorCode DMInitialize_Plex(DM dm)
4484 {
4485   PetscFunctionBegin;
4486   dm->ops->view                      = DMView_Plex;
4487   dm->ops->load                      = DMLoad_Plex;
4488   dm->ops->setfromoptions            = DMSetFromOptions_Plex;
4489   dm->ops->clone                     = DMClone_Plex;
4490   dm->ops->setup                     = DMSetUp_Plex;
4491   dm->ops->createlocalsection        = DMCreateLocalSection_Plex;
4492   dm->ops->createdefaultconstraints  = DMCreateDefaultConstraints_Plex;
4493   dm->ops->createglobalvector        = DMCreateGlobalVector_Plex;
4494   dm->ops->createlocalvector         = DMCreateLocalVector_Plex;
4495   dm->ops->getlocaltoglobalmapping   = NULL;
4496   dm->ops->createfieldis             = NULL;
4497   dm->ops->createcoordinatedm        = DMCreateCoordinateDM_Plex;
4498   dm->ops->createcoordinatefield     = DMCreateCoordinateField_Plex;
4499   dm->ops->getcoloring               = NULL;
4500   dm->ops->creatematrix              = DMCreateMatrix_Plex;
4501   dm->ops->createinterpolation       = DMCreateInterpolation_Plex;
4502   dm->ops->createmassmatrix          = DMCreateMassMatrix_Plex;
4503   dm->ops->createmassmatrixlumped    = DMCreateMassMatrixLumped_Plex;
4504   dm->ops->createinjection           = DMCreateInjection_Plex;
4505   dm->ops->refine                    = DMRefine_Plex;
4506   dm->ops->coarsen                   = DMCoarsen_Plex;
4507   dm->ops->refinehierarchy           = DMRefineHierarchy_Plex;
4508   dm->ops->coarsenhierarchy          = DMCoarsenHierarchy_Plex;
4509   dm->ops->extrude                   = DMExtrude_Plex;
4510   dm->ops->globaltolocalbegin        = NULL;
4511   dm->ops->globaltolocalend          = NULL;
4512   dm->ops->localtoglobalbegin        = NULL;
4513   dm->ops->localtoglobalend          = NULL;
4514   dm->ops->destroy                   = DMDestroy_Plex;
4515   dm->ops->createsubdm               = DMCreateSubDM_Plex;
4516   dm->ops->createsuperdm             = DMCreateSuperDM_Plex;
4517   dm->ops->getdimpoints              = DMGetDimPoints_Plex;
4518   dm->ops->locatepoints              = DMLocatePoints_Plex;
4519   dm->ops->projectfunctionlocal      = DMProjectFunctionLocal_Plex;
4520   dm->ops->projectfunctionlabellocal = DMProjectFunctionLabelLocal_Plex;
4521   dm->ops->projectfieldlocal         = DMProjectFieldLocal_Plex;
4522   dm->ops->projectfieldlabellocal    = DMProjectFieldLabelLocal_Plex;
4523   dm->ops->projectbdfieldlabellocal  = DMProjectBdFieldLabelLocal_Plex;
4524   dm->ops->computel2diff             = DMComputeL2Diff_Plex;
4525   dm->ops->computel2gradientdiff     = DMComputeL2GradientDiff_Plex;
4526   dm->ops->computel2fielddiff        = DMComputeL2FieldDiff_Plex;
4527   dm->ops->getneighbors              = DMGetNeighbors_Plex;
4528   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertBoundaryValues_C", DMPlexInsertBoundaryValues_Plex));
4529   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertTimeDerviativeBoundaryValues_C", DMPlexInsertTimeDerivativeBoundaryValues_Plex));
4530   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMSetUpGLVisViewer_C", DMSetUpGLVisViewer_Plex));
4531   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMCreateNeumannOverlap_C", DMCreateNeumannOverlap_Plex));
4532   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4533   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeGetDefault_C", DMPlexDistributeGetDefault_Plex));
4534   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeSetDefault_C", DMPlexDistributeSetDefault_Plex));
4535   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderGetDefault_C", DMPlexReorderGetDefault_Plex));
4536   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderSetDefault_C", DMPlexReorderSetDefault_Plex));
4537   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMInterpolateSolution_C", DMInterpolateSolution_Plex));
4538   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4539   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetOverlap_C", DMPlexSetOverlap_Plex));
4540   PetscFunctionReturn(0);
4541 }
4542 
4543 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
4544 {
4545   DM_Plex *mesh = (DM_Plex *)dm->data;
4546   PetscSF  face_sf;
4547 
4548   PetscFunctionBegin;
4549   mesh->refct++;
4550   (*newdm)->data = mesh;
4551   PetscCall(DMPlexGetIsoperiodicFaceSF(dm, &face_sf));
4552   PetscCall(DMPlexSetIsoperiodicFaceSF(*newdm, face_sf));
4553   PetscCall(PetscObjectChangeTypeName((PetscObject)*newdm, DMPLEX));
4554   PetscCall(DMInitialize_Plex(*newdm));
4555   PetscFunctionReturn(0);
4556 }
4557 
4558 /*MC
4559   DMPLEX = "plex" - A `DM` object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
4560                     In the local representation, Vecs contain all unknowns in the interior and shared boundary. This is
4561                     specified by a PetscSection object. Ownership in the global representation is determined by
4562                     ownership of the underlying `DMPLEX` points. This is specified by another `PetscSection` object.
4563 
4564   Options Database Keys:
4565 + -dm_refine_pre                     - Refine mesh before distribution
4566 + -dm_refine_uniform_pre             - Choose uniform or generator-based refinement
4567 + -dm_refine_volume_limit_pre        - Cell volume limit after pre-refinement using generator
4568 . -dm_distribute                     - Distribute mesh across processes
4569 . -dm_distribute_overlap             - Number of cells to overlap for distribution
4570 . -dm_refine                         - Refine mesh after distribution
4571 . -dm_plex_hash_location             - Use grid hashing for point location
4572 . -dm_plex_hash_box_faces <n,m,p>    - The number of divisions in each direction of the grid hash
4573 . -dm_plex_partition_balance         - Attempt to evenly divide points on partition boundary between processes
4574 . -dm_plex_remesh_bd                 - Allow changes to the boundary on remeshing
4575 . -dm_plex_max_projection_height     - Maximum mesh point height used to project locally
4576 . -dm_plex_regular_refinement        - Use special nested projection algorithm for regular refinement
4577 . -dm_plex_check_all                 - Perform all shecks below
4578 . -dm_plex_check_symmetry            - Check that the adjacency information in the mesh is symmetric
4579 . -dm_plex_check_skeleton <celltype> - Check that each cell has the correct number of vertices
4580 . -dm_plex_check_faces <celltype>    - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
4581 . -dm_plex_check_geometry            - Check that cells have positive volume
4582 . -dm_view :mesh.tex:ascii_latex     - View the mesh in LaTeX/TikZ
4583 . -dm_plex_view_scale <num>          - Scale the TikZ
4584 - -dm_plex_print_fem <num>           - View FEM assembly information, such as element vectors and matrices
4585 
4586   Level: intermediate
4587 
4588 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`, `PetscSection`
4589 M*/
4590 
4591 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
4592 {
4593   DM_Plex *mesh;
4594   PetscInt unit;
4595 
4596   PetscFunctionBegin;
4597   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4598   PetscCall(PetscNew(&mesh));
4599   dm->data = mesh;
4600 
4601   mesh->refct = 1;
4602   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection));
4603   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection));
4604   mesh->refinementUniform      = PETSC_TRUE;
4605   mesh->refinementLimit        = -1.0;
4606   mesh->distDefault            = PETSC_TRUE;
4607   mesh->reorderDefault         = DMPLEX_REORDER_DEFAULT_NOTSET;
4608   mesh->distributionName       = NULL;
4609   mesh->interpolated           = DMPLEX_INTERPOLATED_INVALID;
4610   mesh->interpolatedCollective = DMPLEX_INTERPOLATED_INVALID;
4611 
4612   PetscCall(PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner));
4613   mesh->remeshBd = PETSC_FALSE;
4614 
4615   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
4616 
4617   mesh->depthState    = -1;
4618   mesh->celltypeState = -1;
4619   mesh->printTol      = 1.0e-10;
4620 
4621   PetscCall(DMInitialize_Plex(dm));
4622   PetscFunctionReturn(0);
4623 }
4624 
4625 /*@
4626   DMPlexCreate - Creates a `DMPLEX` object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
4627 
4628   Collective
4629 
4630   Input Parameter:
4631 . comm - The communicator for the `DMPLEX` object
4632 
4633   Output Parameter:
4634 . mesh  - The `DMPLEX` object
4635 
4636   Level: beginner
4637 
4638 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`
4639 @*/
4640 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
4641 {
4642   PetscFunctionBegin;
4643   PetscValidPointer(mesh, 2);
4644   PetscCall(DMCreate(comm, mesh));
4645   PetscCall(DMSetType(*mesh, DMPLEX));
4646   PetscFunctionReturn(0);
4647 }
4648 
4649 /*@C
4650   DMPlexBuildFromCellListParallel - Build distributed `DMPLEX` topology from a list of vertices for each cell (common mesh generator output)
4651 
4652   Collective on dm
4653 
4654   Input Parameters:
4655 + dm - The `DM`
4656 . numCells - The number of cells owned by this process
4657 . numVertices - The number of vertices to be owned by this process, or `PETSC_DECIDE`
4658 . NVertices - The global number of vertices, or `PETSC_DETERMINE`
4659 . numCorners - The number of vertices for each cell
4660 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4661 
4662   Output Parameters:
4663 + vertexSF - (Optional) `PetscSF` describing complete vertex ownership
4664 - verticesAdjSaved - (Optional) vertex adjacency array
4665 
4666   Level: advanced
4667 
4668   Notes:
4669   Two triangles sharing a face
4670 .vb
4671 
4672         2
4673       / | \
4674      /  |  \
4675     /   |   \
4676    0  0 | 1  3
4677     \   |   /
4678      \  |  /
4679       \ | /
4680         1
4681 .ve
4682 would have input
4683 .vb
4684   numCells = 2, numVertices = 4
4685   cells = [0 1 2  1 3 2]
4686 .ve
4687 which would result in the `DMPLEX`
4688 .vb
4689 
4690         4
4691       / | \
4692      /  |  \
4693     /   |   \
4694    2  0 | 1  5
4695     \   |   /
4696      \  |  /
4697       \ | /
4698         3
4699 .ve
4700 
4701   Vertices are implicitly numbered consecutively 0,...,NVertices.
4702   Each rank owns a chunk of numVertices consecutive vertices.
4703   If numVertices is `PETSC_DECIDE`, PETSc will distribute them as evenly as possible using PetscLayout.
4704   If NVertices is `PETSC_DETERMINE` and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
4705   If only NVertices is `PETSC_DETERMINE`, it is computed as the sum of numVertices over all ranks.
4706 
4707   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
4708 
4709   Fortran Note:
4710   Not currently supported in Fortran.
4711 
4712 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildCoordinatesFromCellListParallel()`,
4713           `PetscSF`
4714 @*/
4715 PetscErrorCode DMPlexBuildFromCellListParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, const PetscInt cells[], PetscSF *vertexSF, PetscInt **verticesAdjSaved)
4716 {
4717   PetscSF     sfPoint;
4718   PetscLayout layout;
4719   PetscInt    numVerticesAdj, *verticesAdj, *cones, c, p;
4720 
4721   PetscFunctionBegin;
4722   PetscValidLogicalCollectiveInt(dm, NVertices, 4);
4723   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4724   /* Get/check global number of vertices */
4725   {
4726     PetscInt       NVerticesInCells, i;
4727     const PetscInt len = numCells * numCorners;
4728 
4729     /* NVerticesInCells = max(cells) + 1 */
4730     NVerticesInCells = PETSC_MIN_INT;
4731     for (i = 0; i < len; i++)
4732       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4733     ++NVerticesInCells;
4734     PetscCallMPI(MPI_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
4735 
4736     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
4737     else
4738       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
4739   }
4740   /* Count locally unique vertices */
4741   {
4742     PetscHSetI vhash;
4743     PetscInt   off = 0;
4744 
4745     PetscCall(PetscHSetICreate(&vhash));
4746     for (c = 0; c < numCells; ++c) {
4747       for (p = 0; p < numCorners; ++p) PetscCall(PetscHSetIAdd(vhash, cells[c * numCorners + p]));
4748     }
4749     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
4750     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
4751     else verticesAdj = *verticesAdjSaved;
4752     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
4753     PetscCall(PetscHSetIDestroy(&vhash));
4754     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
4755   }
4756   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
4757   /* Create cones */
4758   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
4759   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
4760   PetscCall(DMSetUp(dm));
4761   PetscCall(DMPlexGetCones(dm, &cones));
4762   for (c = 0; c < numCells; ++c) {
4763     for (p = 0; p < numCorners; ++p) {
4764       const PetscInt gv = cells[c * numCorners + p];
4765       PetscInt       lv;
4766 
4767       /* Positions within verticesAdj form 0-based local vertex numbering;
4768          we need to shift it by numCells to get correct DAG points (cells go first) */
4769       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
4770       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
4771       cones[c * numCorners + p] = lv + numCells;
4772     }
4773   }
4774   /* Build point sf */
4775   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
4776   PetscCall(PetscLayoutSetSize(layout, NVertices));
4777   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
4778   PetscCall(PetscLayoutSetBlockSize(layout, 1));
4779   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
4780   PetscCall(PetscLayoutDestroy(&layout));
4781   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
4782   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
4783   if (dm->sf) {
4784     const char *prefix;
4785 
4786     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
4787     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
4788   }
4789   PetscCall(DMSetPointSF(dm, sfPoint));
4790   PetscCall(PetscSFDestroy(&sfPoint));
4791   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)(*vertexSF), "Vertex Ownership SF"));
4792   /* Fill in the rest of the topology structure */
4793   PetscCall(DMPlexSymmetrize(dm));
4794   PetscCall(DMPlexStratify(dm));
4795   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4796   PetscFunctionReturn(0);
4797 }
4798 
4799 /*@C
4800   DMPlexBuildCoordinatesFromCellListParallel - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4801 
4802   Collective on dm
4803 
4804   Input Parameters:
4805 + dm - The `DM`
4806 . spaceDim - The spatial dimension used for coordinates
4807 . sfVert - `PetscSF` describing complete vertex ownership
4808 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4809 
4810   Level: advanced
4811 
4812   Fortran Note:
4813   Not currently supported in Fortran.
4814 
4815 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellListParallel()`
4816 @*/
4817 PetscErrorCode DMPlexBuildCoordinatesFromCellListParallel(DM dm, PetscInt spaceDim, PetscSF sfVert, const PetscReal vertexCoords[])
4818 {
4819   PetscSection coordSection;
4820   Vec          coordinates;
4821   PetscScalar *coords;
4822   PetscInt     numVertices, numVerticesAdj, coordSize, v, vStart, vEnd;
4823 
4824   PetscFunctionBegin;
4825   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4826   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
4827   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4828   PetscCall(DMSetCoordinateDim(dm, spaceDim));
4829   PetscCall(PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL));
4830   PetscCheck(vEnd - vStart == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Supplied sfVert has wrong number of leaves = %" PetscInt_FMT " != %" PetscInt_FMT " = vEnd - vStart", numVerticesAdj, vEnd - vStart);
4831   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4832   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4833   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
4834   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
4835   for (v = vStart; v < vEnd; ++v) {
4836     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
4837     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
4838   }
4839   PetscCall(PetscSectionSetUp(coordSection));
4840   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4841   PetscCall(VecCreate(PetscObjectComm((PetscObject)dm), &coordinates));
4842   PetscCall(VecSetBlockSize(coordinates, spaceDim));
4843   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4844   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4845   PetscCall(VecSetType(coordinates, VECSTANDARD));
4846   PetscCall(VecGetArray(coordinates, &coords));
4847   {
4848     MPI_Datatype coordtype;
4849 
4850     /* Need a temp buffer for coords if we have complex/single */
4851     PetscCallMPI(MPI_Type_contiguous(spaceDim, MPIU_SCALAR, &coordtype));
4852     PetscCallMPI(MPI_Type_commit(&coordtype));
4853 #if defined(PETSC_USE_COMPLEX)
4854     {
4855       PetscScalar *svertexCoords;
4856       PetscInt     i;
4857       PetscCall(PetscMalloc1(numVertices * spaceDim, &svertexCoords));
4858       for (i = 0; i < numVertices * spaceDim; i++) svertexCoords[i] = vertexCoords[i];
4859       PetscCall(PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4860       PetscCall(PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4861       PetscCall(PetscFree(svertexCoords));
4862     }
4863 #else
4864     PetscCall(PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4865     PetscCall(PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4866 #endif
4867     PetscCallMPI(MPI_Type_free(&coordtype));
4868   }
4869   PetscCall(VecRestoreArray(coordinates, &coords));
4870   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4871   PetscCall(VecDestroy(&coordinates));
4872   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4873   PetscFunctionReturn(0);
4874 }
4875 
4876 /*@
4877   DMPlexCreateFromCellListParallelPetsc - Create distributed `DMPLEX` from a list of vertices for each cell (common mesh generator output)
4878 
4879   Collective
4880 
4881   Input Parameters:
4882 + comm - The communicator
4883 . dim - The topological dimension of the mesh
4884 . numCells - The number of cells owned by this process
4885 . numVertices - The number of vertices owned by this process, or `PETSC_DECIDE`
4886 . NVertices - The global number of vertices, or `PETSC_DECIDE`
4887 . numCorners - The number of vertices for each cell
4888 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4889 . cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4890 . spaceDim - The spatial dimension used for coordinates
4891 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4892 
4893   Output Parameters:
4894 + dm - The `DM`
4895 . vertexSF - (Optional) `PetscSF` describing complete vertex ownership
4896 - verticesAdjSaved - (Optional) vertex adjacency array
4897 
4898   Level: intermediate
4899 
4900   Notes:
4901   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`,
4902   `DMPlexBuildFromCellListParallel()`, `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellListParallel()`
4903 
4904   See `DMPlexBuildFromCellListParallel()` for an example and details about the topology-related parameters.
4905 
4906   See `DMPlexBuildCoordinatesFromCellListParallel()` for details about the geometry-related parameters.
4907 
4908 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
4909 @*/
4910 PetscErrorCode DMPlexCreateFromCellListParallelPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, PetscInt **verticesAdj, DM *dm)
4911 {
4912   PetscSF sfVert;
4913 
4914   PetscFunctionBegin;
4915   PetscCall(DMCreate(comm, dm));
4916   PetscCall(DMSetType(*dm, DMPLEX));
4917   PetscValidLogicalCollectiveInt(*dm, dim, 2);
4918   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
4919   PetscCall(DMSetDimension(*dm, dim));
4920   PetscCall(DMPlexBuildFromCellListParallel(*dm, numCells, numVertices, NVertices, numCorners, cells, &sfVert, verticesAdj));
4921   if (interpolate) {
4922     DM idm;
4923 
4924     PetscCall(DMPlexInterpolate(*dm, &idm));
4925     PetscCall(DMDestroy(dm));
4926     *dm = idm;
4927   }
4928   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
4929   if (vertexSF) *vertexSF = sfVert;
4930   else PetscCall(PetscSFDestroy(&sfVert));
4931   PetscFunctionReturn(0);
4932 }
4933 
4934 /*@C
4935   DMPlexBuildFromCellList - Build `DMPLEX` topology from a list of vertices for each cell (common mesh generator output)
4936 
4937   Collective on dm
4938 
4939   Input Parameters:
4940 + dm - The `DM`
4941 . numCells - The number of cells owned by this process
4942 . numVertices - The number of vertices owned by this process, or `PETSC_DETERMINE`
4943 . numCorners - The number of vertices for each cell
4944 - cells - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4945 
4946   Level: advanced
4947 
4948   Notes:
4949   Two triangles sharing a face
4950 .vb
4951 
4952         2
4953       / | \
4954      /  |  \
4955     /   |   \
4956    0  0 | 1  3
4957     \   |   /
4958      \  |  /
4959       \ | /
4960         1
4961 .ve
4962 would have input
4963 .vb
4964   numCells = 2, numVertices = 4
4965   cells = [0 1 2  1 3 2]
4966 .ve
4967 which would result in the `DMPLEX`
4968 .vb
4969 
4970         4
4971       / | \
4972      /  |  \
4973     /   |   \
4974    2  0 | 1  5
4975     \   |   /
4976      \  |  /
4977       \ | /
4978         3
4979 .ve
4980 
4981   If numVertices is `PETSC_DETERMINE`, it is computed by PETSc as the maximum vertex index in cells + 1.
4982 
4983   Not currently supported in Fortran.
4984 
4985 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListPetsc()`
4986 @*/
4987 PetscErrorCode DMPlexBuildFromCellList(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const PetscInt cells[])
4988 {
4989   PetscInt *cones, c, p, dim;
4990 
4991   PetscFunctionBegin;
4992   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4993   PetscCall(DMGetDimension(dm, &dim));
4994   /* Get/check global number of vertices */
4995   {
4996     PetscInt       NVerticesInCells, i;
4997     const PetscInt len = numCells * numCorners;
4998 
4999     /* NVerticesInCells = max(cells) + 1 */
5000     NVerticesInCells = PETSC_MIN_INT;
5001     for (i = 0; i < len; i++)
5002       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
5003     ++NVerticesInCells;
5004 
5005     if (numVertices == PETSC_DECIDE) numVertices = NVerticesInCells;
5006     else
5007       PetscCheck(numVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, numVertices, NVerticesInCells);
5008   }
5009   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
5010   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
5011   PetscCall(DMSetUp(dm));
5012   PetscCall(DMPlexGetCones(dm, &cones));
5013   for (c = 0; c < numCells; ++c) {
5014     for (p = 0; p < numCorners; ++p) cones[c * numCorners + p] = cells[c * numCorners + p] + numCells;
5015   }
5016   PetscCall(DMPlexSymmetrize(dm));
5017   PetscCall(DMPlexStratify(dm));
5018   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5019   PetscFunctionReturn(0);
5020 }
5021 
5022 /*@C
5023   DMPlexBuildCoordinatesFromCellList - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
5024 
5025   Collective on dm
5026 
5027   Input Parameters:
5028 + dm - The `DM`
5029 . spaceDim - The spatial dimension used for coordinates
5030 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
5031 
5032   Level: advanced
5033 
5034   Fortran Note:
5035   Not currently supported in Fortran.
5036 
5037 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellList()`
5038 @*/
5039 PetscErrorCode DMPlexBuildCoordinatesFromCellList(DM dm, PetscInt spaceDim, const PetscReal vertexCoords[])
5040 {
5041   PetscSection coordSection;
5042   Vec          coordinates;
5043   DM           cdm;
5044   PetscScalar *coords;
5045   PetscInt     v, vStart, vEnd, d;
5046 
5047   PetscFunctionBegin;
5048   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
5049   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
5050   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
5051   PetscCall(DMSetCoordinateDim(dm, spaceDim));
5052   PetscCall(DMGetCoordinateSection(dm, &coordSection));
5053   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5054   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
5055   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
5056   for (v = vStart; v < vEnd; ++v) {
5057     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
5058     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
5059   }
5060   PetscCall(PetscSectionSetUp(coordSection));
5061 
5062   PetscCall(DMGetCoordinateDM(dm, &cdm));
5063   PetscCall(DMCreateLocalVector(cdm, &coordinates));
5064   PetscCall(VecSetBlockSize(coordinates, spaceDim));
5065   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5066   PetscCall(VecGetArrayWrite(coordinates, &coords));
5067   for (v = 0; v < vEnd - vStart; ++v) {
5068     for (d = 0; d < spaceDim; ++d) coords[v * spaceDim + d] = vertexCoords[v * spaceDim + d];
5069   }
5070   PetscCall(VecRestoreArrayWrite(coordinates, &coords));
5071   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
5072   PetscCall(VecDestroy(&coordinates));
5073   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
5074   PetscFunctionReturn(0);
5075 }
5076 
5077 /*@
5078   DMPlexCreateFromCellListPetsc - Create `DMPLEX` from a list of vertices for each cell (common mesh generator output), but only process 0 takes in the input
5079 
5080   Collective
5081 
5082   Input Parameters:
5083 + comm - The communicator
5084 . dim - The topological dimension of the mesh
5085 . numCells - The number of cells, only on process 0
5086 . numVertices - The number of vertices owned by this process, or `PETSC_DECIDE`, only on process 0
5087 . numCorners - The number of vertices for each cell, only on process 0
5088 . interpolate - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
5089 . cells - An array of numCells*numCorners numbers, the vertices for each cell, only on process 0
5090 . spaceDim - The spatial dimension used for coordinates
5091 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex, only on process 0
5092 
5093   Output Parameter:
5094 . dm - The `DM`, which only has points on process 0
5095 
5096   Level: intermediate
5097 
5098   Notes:
5099   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`, `DMPlexBuildFromCellList()`,
5100   `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellList()`
5101 
5102   See `DMPlexBuildFromCellList()` for an example and details about the topology-related parameters.
5103   See `DMPlexBuildCoordinatesFromCellList()` for details about the geometry-related parameters.
5104   See `DMPlexCreateFromCellListParallelPetsc()` for parallel input
5105 
5106 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellList()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
5107 @*/
5108 PetscErrorCode DMPlexCreateFromCellListPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], DM *dm)
5109 {
5110   PetscMPIInt rank;
5111 
5112   PetscFunctionBegin;
5113   PetscCheck(dim, comm, PETSC_ERR_ARG_OUTOFRANGE, "This is not appropriate for 0-dimensional meshes. Consider either creating the DM using DMPlexCreateFromDAG(), by hand, or using DMSwarm.");
5114   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5115   PetscCall(DMCreate(comm, dm));
5116   PetscCall(DMSetType(*dm, DMPLEX));
5117   PetscCall(DMSetDimension(*dm, dim));
5118   if (rank == 0) PetscCall(DMPlexBuildFromCellList(*dm, numCells, numVertices, numCorners, cells));
5119   else PetscCall(DMPlexBuildFromCellList(*dm, 0, 0, 0, NULL));
5120   if (interpolate) {
5121     DM idm;
5122 
5123     PetscCall(DMPlexInterpolate(*dm, &idm));
5124     PetscCall(DMDestroy(dm));
5125     *dm = idm;
5126   }
5127   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, vertexCoords));
5128   else PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, NULL));
5129   PetscFunctionReturn(0);
5130 }
5131 
5132 /*@
5133   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a DM
5134 
5135   Input Parameters:
5136 + dm - The empty DM object, usually from DMCreate() and DMSetDimension()
5137 . depth - The depth of the DAG
5138 . numPoints - Array of size depth + 1 containing the number of points at each depth
5139 . coneSize - The cone size of each point
5140 . cones - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
5141 . coneOrientations - The orientation of each cone point
5142 - vertexCoords - An array of numPoints[0]*spacedim numbers representing the coordinates of each vertex, with spacedim the value set via DMSetCoordinateDim()
5143 
5144   Output Parameter:
5145 . dm - The DM
5146 
5147   Note:
5148   Two triangles sharing a face would have input
5149 .vb
5150   depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
5151   cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
5152  vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
5153 .ve
5154 which would result in the DMPlex
5155 .vb
5156         4
5157       / | \
5158      /  |  \
5159     /   |   \
5160    2  0 | 1  5
5161     \   |   /
5162      \  |  /
5163       \ | /
5164         3
5165 .ve
5166  Notice that all points are numbered consecutively, unlike `DMPlexCreateFromCellListPetsc()`
5167 
5168   Level: advanced
5169 
5170 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
5171 @*/
5172 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
5173 {
5174   Vec          coordinates;
5175   PetscSection coordSection;
5176   PetscScalar *coords;
5177   PetscInt     coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
5178 
5179   PetscFunctionBegin;
5180   PetscCall(DMGetDimension(dm, &dim));
5181   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
5182   PetscCheck(dimEmbed >= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Embedding dimension %" PetscInt_FMT " cannot be less than intrinsic dimension %" PetscInt_FMT, dimEmbed, dim);
5183   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
5184   PetscCall(DMPlexSetChart(dm, pStart, pEnd));
5185   for (p = pStart; p < pEnd; ++p) {
5186     PetscCall(DMPlexSetConeSize(dm, p, coneSize[p - pStart]));
5187     if (firstVertex < 0 && !coneSize[p - pStart]) firstVertex = p - pStart;
5188   }
5189   PetscCheck(firstVertex >= 0 || !numPoints[0], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Expected %" PetscInt_FMT " vertices but could not find any", numPoints[0]);
5190   PetscCall(DMSetUp(dm)); /* Allocate space for cones */
5191   for (p = pStart, off = 0; p < pEnd; off += coneSize[p - pStart], ++p) {
5192     PetscCall(DMPlexSetCone(dm, p, &cones[off]));
5193     PetscCall(DMPlexSetConeOrientation(dm, p, &coneOrientations[off]));
5194   }
5195   PetscCall(DMPlexSymmetrize(dm));
5196   PetscCall(DMPlexStratify(dm));
5197   /* Build coordinates */
5198   PetscCall(DMGetCoordinateSection(dm, &coordSection));
5199   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5200   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dimEmbed));
5201   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numPoints[0]));
5202   for (v = firstVertex; v < firstVertex + numPoints[0]; ++v) {
5203     PetscCall(PetscSectionSetDof(coordSection, v, dimEmbed));
5204     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed));
5205   }
5206   PetscCall(PetscSectionSetUp(coordSection));
5207   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
5208   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
5209   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5210   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
5211   PetscCall(VecSetBlockSize(coordinates, dimEmbed));
5212   PetscCall(VecSetType(coordinates, VECSTANDARD));
5213   if (vertexCoords) {
5214     PetscCall(VecGetArray(coordinates, &coords));
5215     for (v = 0; v < numPoints[0]; ++v) {
5216       PetscInt off;
5217 
5218       PetscCall(PetscSectionGetOffset(coordSection, v + firstVertex, &off));
5219       for (d = 0; d < dimEmbed; ++d) coords[off + d] = vertexCoords[v * dimEmbed + d];
5220     }
5221   }
5222   PetscCall(VecRestoreArray(coordinates, &coords));
5223   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
5224   PetscCall(VecDestroy(&coordinates));
5225   PetscFunctionReturn(0);
5226 }
5227 
5228 /*@C
5229   DMPlexCreateCellVertexFromFile - Create a `DMPLEX` mesh from a simple cell-vertex file.
5230 
5231   Collective
5232 
5233 + comm        - The MPI communicator
5234 . filename    - Name of the .dat file
5235 - interpolate - Create faces and edges in the mesh
5236 
5237   Output Parameter:
5238 . dm  - The `DM` object representing the mesh
5239 
5240   Level: beginner
5241 
5242   Note:
5243   The format is the simplest possible:
5244 .vb
5245   Ne
5246   v0 v1 ... vk
5247   Nv
5248   x y z marker
5249 .ve
5250 
5251   Developer Note:
5252   Should use a `PetscViewer` not a filename
5253 
5254 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromFile()`, `DMPlexCreateMedFromFile()`, `DMPlexCreateGmsh()`, `DMPlexCreate()`
5255 @*/
5256 PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
5257 {
5258   DMLabel      marker;
5259   PetscViewer  viewer;
5260   Vec          coordinates;
5261   PetscSection coordSection;
5262   PetscScalar *coords;
5263   char         line[PETSC_MAX_PATH_LEN];
5264   PetscInt     dim = 3, cdim = 3, coordSize, v, c, d;
5265   PetscMPIInt  rank;
5266   int          snum, Nv, Nc, Ncn, Nl;
5267 
5268   PetscFunctionBegin;
5269   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5270   PetscCall(PetscViewerCreate(comm, &viewer));
5271   PetscCall(PetscViewerSetType(viewer, PETSCVIEWERASCII));
5272   PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
5273   PetscCall(PetscViewerFileSetName(viewer, filename));
5274   if (rank == 0) {
5275     PetscCall(PetscViewerRead(viewer, line, 4, NULL, PETSC_STRING));
5276     snum = sscanf(line, "%d %d %d %d", &Nc, &Nv, &Ncn, &Nl);
5277     PetscCheck(snum == 4, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5278   } else {
5279     Nc = Nv = Ncn = Nl = 0;
5280   }
5281   PetscCall(DMCreate(comm, dm));
5282   PetscCall(DMSetType(*dm, DMPLEX));
5283   PetscCall(DMPlexSetChart(*dm, 0, Nc + Nv));
5284   PetscCall(DMSetDimension(*dm, dim));
5285   PetscCall(DMSetCoordinateDim(*dm, cdim));
5286   /* Read topology */
5287   if (rank == 0) {
5288     char     format[PETSC_MAX_PATH_LEN];
5289     PetscInt cone[8];
5290     int      vbuf[8], v;
5291 
5292     for (c = 0; c < Ncn; ++c) {
5293       format[c * 3 + 0] = '%';
5294       format[c * 3 + 1] = 'd';
5295       format[c * 3 + 2] = ' ';
5296     }
5297     format[Ncn * 3 - 1] = '\0';
5298     for (c = 0; c < Nc; ++c) PetscCall(DMPlexSetConeSize(*dm, c, Ncn));
5299     PetscCall(DMSetUp(*dm));
5300     for (c = 0; c < Nc; ++c) {
5301       PetscCall(PetscViewerRead(viewer, line, Ncn, NULL, PETSC_STRING));
5302       switch (Ncn) {
5303       case 2:
5304         snum = sscanf(line, format, &vbuf[0], &vbuf[1]);
5305         break;
5306       case 3:
5307         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2]);
5308         break;
5309       case 4:
5310         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3]);
5311         break;
5312       case 6:
5313         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5]);
5314         break;
5315       case 8:
5316         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);
5317         break;
5318       default:
5319         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No cell shape with %d vertices", Ncn);
5320       }
5321       PetscCheck(snum == Ncn, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5322       for (v = 0; v < Ncn; ++v) cone[v] = vbuf[v] + Nc;
5323       /* Hexahedra are inverted */
5324       if (Ncn == 8) {
5325         PetscInt tmp = cone[1];
5326         cone[1]      = cone[3];
5327         cone[3]      = tmp;
5328       }
5329       PetscCall(DMPlexSetCone(*dm, c, cone));
5330     }
5331   }
5332   PetscCall(DMPlexSymmetrize(*dm));
5333   PetscCall(DMPlexStratify(*dm));
5334   /* Read coordinates */
5335   PetscCall(DMGetCoordinateSection(*dm, &coordSection));
5336   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5337   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
5338   PetscCall(PetscSectionSetChart(coordSection, Nc, Nc + Nv));
5339   for (v = Nc; v < Nc + Nv; ++v) {
5340     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
5341     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
5342   }
5343   PetscCall(PetscSectionSetUp(coordSection));
5344   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
5345   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
5346   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5347   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
5348   PetscCall(VecSetBlockSize(coordinates, cdim));
5349   PetscCall(VecSetType(coordinates, VECSTANDARD));
5350   PetscCall(VecGetArray(coordinates, &coords));
5351   if (rank == 0) {
5352     char   format[PETSC_MAX_PATH_LEN];
5353     double x[3];
5354     int    l, val[3];
5355 
5356     if (Nl) {
5357       for (l = 0; l < Nl; ++l) {
5358         format[l * 3 + 0] = '%';
5359         format[l * 3 + 1] = 'd';
5360         format[l * 3 + 2] = ' ';
5361       }
5362       format[Nl * 3 - 1] = '\0';
5363       PetscCall(DMCreateLabel(*dm, "marker"));
5364       PetscCall(DMGetLabel(*dm, "marker", &marker));
5365     }
5366     for (v = 0; v < Nv; ++v) {
5367       PetscCall(PetscViewerRead(viewer, line, 3 + Nl, NULL, PETSC_STRING));
5368       snum = sscanf(line, "%lg %lg %lg", &x[0], &x[1], &x[2]);
5369       PetscCheck(snum == 3, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5370       switch (Nl) {
5371       case 0:
5372         snum = 0;
5373         break;
5374       case 1:
5375         snum = sscanf(line, format, &val[0]);
5376         break;
5377       case 2:
5378         snum = sscanf(line, format, &val[0], &val[1]);
5379         break;
5380       case 3:
5381         snum = sscanf(line, format, &val[0], &val[1], &val[2]);
5382         break;
5383       default:
5384         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Request support for %d labels", Nl);
5385       }
5386       PetscCheck(snum == Nl, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5387       for (d = 0; d < cdim; ++d) coords[v * cdim + d] = x[d];
5388       for (l = 0; l < Nl; ++l) PetscCall(DMLabelSetValue(marker, v + Nc, val[l]));
5389     }
5390   }
5391   PetscCall(VecRestoreArray(coordinates, &coords));
5392   PetscCall(DMSetCoordinatesLocal(*dm, coordinates));
5393   PetscCall(VecDestroy(&coordinates));
5394   PetscCall(PetscViewerDestroy(&viewer));
5395   if (interpolate) {
5396     DM      idm;
5397     DMLabel bdlabel;
5398 
5399     PetscCall(DMPlexInterpolate(*dm, &idm));
5400     PetscCall(DMDestroy(dm));
5401     *dm = idm;
5402 
5403     if (!Nl) {
5404       PetscCall(DMCreateLabel(*dm, "marker"));
5405       PetscCall(DMGetLabel(*dm, "marker", &bdlabel));
5406       PetscCall(DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel));
5407       PetscCall(DMPlexLabelComplete(*dm, bdlabel));
5408     }
5409   }
5410   PetscFunctionReturn(0);
5411 }
5412 
5413 /*@C
5414   DMPlexCreateFromFile - This takes a filename and produces a `DM`
5415 
5416   Collective
5417 
5418   Input Parameters:
5419 + comm - The communicator
5420 . filename - A file name
5421 . plexname - The object name of the resulting `DM`, also used for intra-datafile lookup by some formats
5422 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
5423 
5424   Output Parameter:
5425 . dm - The `DM`
5426 
5427   Options Database Key:
5428 . -dm_plex_create_from_hdf5_xdmf - use the `PETSC_VIEWER_HDF5_XDMF` format for reading HDF5
5429 
5430   Use -dm_plex_create_ prefix to pass options to the internal PetscViewer, e.g.
5431 $ -dm_plex_create_viewer_hdf5_collective
5432 
5433   Level: beginner
5434 
5435   Notes:
5436   Using `PETSCVIEWERHDF5` type with `PETSC_VIEWER_HDF5_PETSC` format, one can save multiple `DMPLEX`
5437   meshes in a single HDF5 file. This in turn requires one to name the `DMPLEX` object with `PetscObjectSetName()`
5438   before saving it with `DMView()` and before loading it with `DMLoad()` for identification of the mesh object.
5439   The input parameter name is thus used to name the `DMPLEX` object when `DMPlexCreateFromFile()` internally
5440   calls `DMLoad()`. Currently, name is ignored for other viewer types and/or formats.
5441 
5442 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`, `PetscObjectSetName()`, `DMView()`, `DMLoad()`
5443 @*/
5444 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], const char plexname[], PetscBool interpolate, DM *dm)
5445 {
5446   const char  extGmsh[]      = ".msh";
5447   const char  extGmsh2[]     = ".msh2";
5448   const char  extGmsh4[]     = ".msh4";
5449   const char  extCGNS[]      = ".cgns";
5450   const char  extExodus[]    = ".exo";
5451   const char  extExodus_e[]  = ".e";
5452   const char  extGenesis[]   = ".gen";
5453   const char  extFluent[]    = ".cas";
5454   const char  extHDF5[]      = ".h5";
5455   const char  extMed[]       = ".med";
5456   const char  extPLY[]       = ".ply";
5457   const char  extEGADSLite[] = ".egadslite";
5458   const char  extEGADS[]     = ".egads";
5459   const char  extIGES[]      = ".igs";
5460   const char  extSTEP[]      = ".stp";
5461   const char  extCV[]        = ".dat";
5462   size_t      len;
5463   PetscBool   isGmsh, isGmsh2, isGmsh4, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isMed, isPLY, isEGADSLite, isEGADS, isIGES, isSTEP, isCV;
5464   PetscMPIInt rank;
5465 
5466   PetscFunctionBegin;
5467   PetscValidCharPointer(filename, 2);
5468   if (plexname) PetscValidCharPointer(plexname, 3);
5469   PetscValidPointer(dm, 5);
5470   PetscCall(DMInitializePackage());
5471   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5472   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5473   PetscCall(PetscStrlen(filename, &len));
5474   PetscCheck(len, comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
5475 
5476 #define CheckExtension(extension__, is_extension__) \
5477   do { \
5478     PetscAssert(sizeof(extension__), comm, PETSC_ERR_PLIB, "Zero-size extension: %s", extension__); \
5479     /* don't count the null-terminator at the end */ \
5480     const size_t ext_len = sizeof(extension__) - 1; \
5481     if (len < ext_len) { \
5482       is_extension__ = PETSC_FALSE; \
5483     } else { \
5484       PetscCall(PetscStrncmp(filename + len - ext_len, extension__, ext_len, &is_extension__)); \
5485     } \
5486   } while (0)
5487 
5488   CheckExtension(extGmsh, isGmsh);
5489   CheckExtension(extGmsh2, isGmsh2);
5490   CheckExtension(extGmsh4, isGmsh4);
5491   CheckExtension(extCGNS, isCGNS);
5492   CheckExtension(extExodus, isExodus);
5493   if (!isExodus) CheckExtension(extExodus_e, isExodus);
5494   CheckExtension(extGenesis, isGenesis);
5495   CheckExtension(extFluent, isFluent);
5496   CheckExtension(extHDF5, isHDF5);
5497   CheckExtension(extMed, isMed);
5498   CheckExtension(extPLY, isPLY);
5499   CheckExtension(extEGADSLite, isEGADSLite);
5500   CheckExtension(extEGADS, isEGADS);
5501   CheckExtension(extIGES, isIGES);
5502   CheckExtension(extSTEP, isSTEP);
5503   CheckExtension(extCV, isCV);
5504 
5505 #undef CheckExtension
5506 
5507   if (isGmsh || isGmsh2 || isGmsh4) {
5508     PetscCall(DMPlexCreateGmshFromFile(comm, filename, interpolate, dm));
5509   } else if (isCGNS) {
5510     PetscCall(DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm));
5511   } else if (isExodus || isGenesis) {
5512     PetscCall(DMPlexCreateExodusFromFile(comm, filename, interpolate, dm));
5513   } else if (isFluent) {
5514     PetscCall(DMPlexCreateFluentFromFile(comm, filename, interpolate, dm));
5515   } else if (isHDF5) {
5516     PetscBool   load_hdf5_xdmf = PETSC_FALSE;
5517     PetscViewer viewer;
5518 
5519     /* PETSC_VIEWER_HDF5_XDMF is used if the filename ends with .xdmf.h5, or if -dm_plex_create_from_hdf5_xdmf option is present */
5520     PetscCall(PetscStrncmp(&filename[PetscMax(0, len - 8)], ".xdmf", 5, &load_hdf5_xdmf));
5521     PetscCall(PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &load_hdf5_xdmf, NULL));
5522     PetscCall(PetscViewerCreate(comm, &viewer));
5523     PetscCall(PetscViewerSetType(viewer, PETSCVIEWERHDF5));
5524     PetscCall(PetscViewerSetOptionsPrefix(viewer, "dm_plex_create_"));
5525     PetscCall(PetscViewerSetFromOptions(viewer));
5526     PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
5527     PetscCall(PetscViewerFileSetName(viewer, filename));
5528 
5529     PetscCall(DMCreate(comm, dm));
5530     PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5531     PetscCall(DMSetType(*dm, DMPLEX));
5532     if (load_hdf5_xdmf) PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF));
5533     PetscCall(DMLoad(*dm, viewer));
5534     if (load_hdf5_xdmf) PetscCall(PetscViewerPopFormat(viewer));
5535     PetscCall(PetscViewerDestroy(&viewer));
5536 
5537     if (interpolate) {
5538       DM idm;
5539 
5540       PetscCall(DMPlexInterpolate(*dm, &idm));
5541       PetscCall(DMDestroy(dm));
5542       *dm = idm;
5543     }
5544   } else if (isMed) {
5545     PetscCall(DMPlexCreateMedFromFile(comm, filename, interpolate, dm));
5546   } else if (isPLY) {
5547     PetscCall(DMPlexCreatePLYFromFile(comm, filename, interpolate, dm));
5548   } else if (isEGADSLite || isEGADS || isIGES || isSTEP) {
5549     if (isEGADSLite) PetscCall(DMPlexCreateEGADSLiteFromFile(comm, filename, dm));
5550     else PetscCall(DMPlexCreateEGADSFromFile(comm, filename, dm));
5551     if (!interpolate) {
5552       DM udm;
5553 
5554       PetscCall(DMPlexUninterpolate(*dm, &udm));
5555       PetscCall(DMDestroy(dm));
5556       *dm = udm;
5557     }
5558   } else if (isCV) {
5559     PetscCall(DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm));
5560   } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
5561   PetscCall(PetscStrlen(plexname, &len));
5562   if (len) PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5563   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5564   PetscFunctionReturn(0);
5565 }
5566 /*@C
5567   DMPlexCreateEphemeral - This takes a `DMPlexTransform` and a base `DMPlex` and produces an ephemeral `DM`, meaning one that is created on the fly in response to queries.
5568 
5569   Input Parameter:
5570 . tr - The `DMPlexTransform`
5571 
5572   Output Parameter:
5573 . dm - The `DM`
5574 
5575   Notes:
5576   An emphemeral mesh is one that is not stored concretely, as in the default Plex implementation, but rather is produced on the fly in response to queries, using information from the transform and the base mesh.
5577 
5578   Level: beginner
5579 
5580 .seealso: `DMPlexCreateFromFile`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
5581 @*/
5582 PetscErrorCode DMPlexCreateEphemeral(DMPlexTransform tr, DM *dm)
5583 {
5584   DM       bdm;
5585   PetscInt Nl;
5586 
5587   PetscFunctionBegin;
5588   PetscCall(DMCreate(PetscObjectComm((PetscObject)tr), dm));
5589   PetscCall(DMSetType(*dm, DMPLEX));
5590   PetscCall(DMSetFromOptions(*dm));
5591 
5592   PetscCall(PetscObjectReference((PetscObject)tr));
5593   PetscCall(DMPlexTransformDestroy(&((DM_Plex *)(*dm)->data)->tr));
5594   ((DM_Plex *)(*dm)->data)->tr = tr;
5595 
5596   PetscCall(DMPlexTransformGetDM(tr, &bdm));
5597   PetscCall(DMGetNumLabels(bdm, &Nl));
5598   for (PetscInt l = 0; l < Nl; ++l) {
5599     DMLabel     label, labelNew;
5600     const char *lname;
5601     PetscBool   isDepth, isCellType;
5602 
5603     PetscCall(DMGetLabelName(bdm, l, &lname));
5604     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
5605     if (isDepth) continue;
5606     PetscCall(PetscStrcmp(lname, "celltype", &isCellType));
5607     if (isCellType) continue;
5608     PetscCall(DMCreateLabel(*dm, lname));
5609     PetscCall(DMGetLabel(bdm, lname, &label));
5610     PetscCall(DMGetLabel(*dm, lname, &labelNew));
5611     PetscCall(DMLabelSetType(labelNew, DMLABELEPHEMERAL));
5612     PetscCall(DMLabelEphemeralSetLabel(labelNew, label));
5613     PetscCall(DMLabelEphemeralSetTransform(labelNew, tr));
5614     PetscCall(DMLabelSetUp(labelNew));
5615   }
5616   PetscFunctionReturn(0);
5617 }
5618