xref: /petsc/src/dm/impls/plex/plexcreate.c (revision 04c3f3b8f563eff258c1de90d4e02d41e6027585)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h> /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>   /*I   "petscdmplex.h"   I*/
4 #include <petscsf.h>
5 #include <petscdmplextransform.h>
6 #include <petscdmlabelephemeral.h>
7 #include <petsc/private/kernels/blockmatmult.h>
8 #include <petsc/private/kernels/blockinvert.h>
9 
10 PetscLogEvent DMPLEX_CreateFromFile, DMPLEX_CreateFromOptions, DMPLEX_BuildFromCellList, DMPLEX_BuildCoordinatesFromCellList;
11 
12 /* External function declarations here */
13 static PetscErrorCode DMInitialize_Plex(DM dm);
14 
15 /* This copies internal things in the Plex structure that we generally want when making a new, related Plex */
16 PetscErrorCode DMPlexCopy_Internal(DM dmin, PetscBool copyPeriodicity, PetscBool copyOverlap, DM dmout)
17 {
18   const PetscReal         *maxCell, *Lstart, *L;
19   PetscBool                dist;
20   DMPlexReorderDefaultFlag reorder;
21 
22   PetscFunctionBegin;
23   if (copyPeriodicity) {
24     PetscCall(DMGetPeriodicity(dmin, &maxCell, &Lstart, &L));
25     PetscCall(DMSetPeriodicity(dmout, maxCell, Lstart, L));
26   }
27   PetscCall(DMPlexDistributeGetDefault(dmin, &dist));
28   PetscCall(DMPlexDistributeSetDefault(dmout, dist));
29   PetscCall(DMPlexReorderGetDefault(dmin, &reorder));
30   PetscCall(DMPlexReorderSetDefault(dmout, reorder));
31   ((DM_Plex *)dmout->data)->useHashLocation = ((DM_Plex *)dmin->data)->useHashLocation;
32   if (copyOverlap) PetscCall(DMPlexSetOverlap_Plex(dmout, dmin, 0));
33   PetscFunctionReturn(PETSC_SUCCESS);
34 }
35 
36 /* Replace dm with the contents of ndm, and then destroy ndm
37    - Share the DM_Plex structure
38    - Share the coordinates
39    - Share the SF
40 */
41 PetscErrorCode DMPlexReplace_Internal(DM dm, DM *ndm)
42 {
43   PetscSF          sf;
44   DM               dmNew = *ndm, coordDM, coarseDM;
45   Vec              coords;
46   const PetscReal *maxCell, *Lstart, *L;
47   PetscInt         dim, cdim;
48 
49   PetscFunctionBegin;
50   if (dm == dmNew) {
51     PetscCall(DMDestroy(ndm));
52     PetscFunctionReturn(PETSC_SUCCESS);
53   }
54   dm->setupcalled = dmNew->setupcalled;
55   PetscCall(DMGetDimension(dmNew, &dim));
56   PetscCall(DMSetDimension(dm, dim));
57   PetscCall(DMGetCoordinateDim(dmNew, &cdim));
58   PetscCall(DMSetCoordinateDim(dm, cdim));
59   PetscCall(DMGetPointSF(dmNew, &sf));
60   PetscCall(DMSetPointSF(dm, sf));
61   PetscCall(DMGetCoordinateDM(dmNew, &coordDM));
62   PetscCall(DMGetCoordinatesLocal(dmNew, &coords));
63   PetscCall(DMSetCoordinateDM(dm, coordDM));
64   PetscCall(DMSetCoordinatesLocal(dm, coords));
65   PetscCall(DMGetCellCoordinateDM(dmNew, &coordDM));
66   PetscCall(DMGetCellCoordinatesLocal(dmNew, &coords));
67   PetscCall(DMSetCellCoordinateDM(dm, coordDM));
68   PetscCall(DMSetCellCoordinatesLocal(dm, coords));
69   /* Do not want to create the coordinate field if it does not already exist, so do not call DMGetCoordinateField() */
70   PetscCall(DMFieldDestroy(&dm->coordinates[0].field));
71   dm->coordinates[0].field            = dmNew->coordinates[0].field;
72   ((DM_Plex *)dmNew->data)->coordFunc = ((DM_Plex *)dm->data)->coordFunc;
73   PetscCall(DMGetPeriodicity(dmNew, &maxCell, &Lstart, &L));
74   PetscCall(DMSetPeriodicity(dm, maxCell, Lstart, L));
75   PetscCall(DMDestroy_Plex(dm));
76   PetscCall(DMInitialize_Plex(dm));
77   dm->data = dmNew->data;
78   ((DM_Plex *)dmNew->data)->refct++;
79   PetscCall(DMPlexGetIsoperiodicFaceSF(dm, &sf));
80   PetscCall(DMPlexSetIsoperiodicFaceSF(dm, sf)); // for the compose function effect on dm
81   PetscCall(DMDestroyLabelLinkList_Internal(dm));
82   PetscCall(DMCopyLabels(dmNew, dm, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL));
83   PetscCall(DMGetCoarseDM(dmNew, &coarseDM));
84   PetscCall(DMSetCoarseDM(dm, coarseDM));
85   PetscCall(DMDestroy(ndm));
86   PetscFunctionReturn(PETSC_SUCCESS);
87 }
88 
89 /* Swap dm with the contents of dmNew
90    - Swap the DM_Plex structure
91    - Swap the coordinates
92    - Swap the point PetscSF
93 */
94 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
95 {
96   DM          coordDMA, coordDMB;
97   Vec         coordsA, coordsB;
98   PetscSF     sfA, sfB;
99   DMField     fieldTmp;
100   void       *tmp;
101   DMLabelLink listTmp;
102   DMLabel     depthTmp;
103   PetscInt    tmpI;
104 
105   PetscFunctionBegin;
106   if (dmA == dmB) PetscFunctionReturn(PETSC_SUCCESS);
107   PetscCall(DMGetPointSF(dmA, &sfA));
108   PetscCall(DMGetPointSF(dmB, &sfB));
109   PetscCall(PetscObjectReference((PetscObject)sfA));
110   PetscCall(DMSetPointSF(dmA, sfB));
111   PetscCall(DMSetPointSF(dmB, sfA));
112   PetscCall(PetscObjectDereference((PetscObject)sfA));
113 
114   PetscCall(DMGetCoordinateDM(dmA, &coordDMA));
115   PetscCall(DMGetCoordinateDM(dmB, &coordDMB));
116   PetscCall(PetscObjectReference((PetscObject)coordDMA));
117   PetscCall(DMSetCoordinateDM(dmA, coordDMB));
118   PetscCall(DMSetCoordinateDM(dmB, coordDMA));
119   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
120 
121   PetscCall(DMGetCoordinatesLocal(dmA, &coordsA));
122   PetscCall(DMGetCoordinatesLocal(dmB, &coordsB));
123   PetscCall(PetscObjectReference((PetscObject)coordsA));
124   PetscCall(DMSetCoordinatesLocal(dmA, coordsB));
125   PetscCall(DMSetCoordinatesLocal(dmB, coordsA));
126   PetscCall(PetscObjectDereference((PetscObject)coordsA));
127 
128   PetscCall(DMGetCellCoordinateDM(dmA, &coordDMA));
129   PetscCall(DMGetCellCoordinateDM(dmB, &coordDMB));
130   PetscCall(PetscObjectReference((PetscObject)coordDMA));
131   PetscCall(DMSetCellCoordinateDM(dmA, coordDMB));
132   PetscCall(DMSetCellCoordinateDM(dmB, coordDMA));
133   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
134 
135   PetscCall(DMGetCellCoordinatesLocal(dmA, &coordsA));
136   PetscCall(DMGetCellCoordinatesLocal(dmB, &coordsB));
137   PetscCall(PetscObjectReference((PetscObject)coordsA));
138   PetscCall(DMSetCellCoordinatesLocal(dmA, coordsB));
139   PetscCall(DMSetCellCoordinatesLocal(dmB, coordsA));
140   PetscCall(PetscObjectDereference((PetscObject)coordsA));
141 
142   fieldTmp                  = dmA->coordinates[0].field;
143   dmA->coordinates[0].field = dmB->coordinates[0].field;
144   dmB->coordinates[0].field = fieldTmp;
145   fieldTmp                  = dmA->coordinates[1].field;
146   dmA->coordinates[1].field = dmB->coordinates[1].field;
147   dmB->coordinates[1].field = fieldTmp;
148   tmp                       = dmA->data;
149   dmA->data                 = dmB->data;
150   dmB->data                 = tmp;
151   listTmp                   = dmA->labels;
152   dmA->labels               = dmB->labels;
153   dmB->labels               = listTmp;
154   depthTmp                  = dmA->depthLabel;
155   dmA->depthLabel           = dmB->depthLabel;
156   dmB->depthLabel           = depthTmp;
157   depthTmp                  = dmA->celltypeLabel;
158   dmA->celltypeLabel        = dmB->celltypeLabel;
159   dmB->celltypeLabel        = depthTmp;
160   tmpI                      = dmA->levelup;
161   dmA->levelup              = dmB->levelup;
162   dmB->levelup              = tmpI;
163   PetscFunctionReturn(PETSC_SUCCESS);
164 }
165 
166 PetscErrorCode DMPlexInterpolateInPlace_Internal(DM dm)
167 {
168   DM idm;
169 
170   PetscFunctionBegin;
171   PetscCall(DMPlexInterpolate(dm, &idm));
172   PetscCall(DMPlexCopyCoordinates(dm, idm));
173   PetscCall(DMPlexReplace_Internal(dm, &idm));
174   PetscFunctionReturn(PETSC_SUCCESS);
175 }
176 
177 /*@C
178   DMPlexCreateCoordinateSpace - Creates a finite element space for the coordinates
179 
180   Collective
181 
182   Input Parameters:
183 + dm        - The `DMPLEX`
184 . degree    - The degree of the finite element or `PETSC_DECIDE`
185 - coordFunc - An optional function to map new points from refinement to the surface
186 
187   Level: advanced
188 
189 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `PetscPointFunc`, `PetscFECreateLagrange()`, `DMGetCoordinateDM()`
190 @*/
191 PetscErrorCode DMPlexCreateCoordinateSpace(DM dm, PetscInt degree, PetscPointFunc coordFunc)
192 {
193   DM_Plex     *mesh = (DM_Plex *)dm->data;
194   DM           cdm;
195   PetscDS      cds;
196   PetscFE      fe;
197   PetscClassId id;
198 
199   PetscFunctionBegin;
200   PetscCall(DMGetCoordinateDM(dm, &cdm));
201   PetscCall(DMGetDS(cdm, &cds));
202   PetscCall(PetscDSGetDiscretization(cds, 0, (PetscObject *)&fe));
203   PetscCall(PetscObjectGetClassId((PetscObject)fe, &id));
204   if (id != PETSCFE_CLASSID) {
205     PetscInt dim, dE, qorder;
206 
207     PetscCall(DMGetDimension(dm, &dim));
208     PetscCall(DMGetCoordinateDim(dm, &dE));
209     qorder = degree;
210     PetscObjectOptionsBegin((PetscObject)cdm);
211     PetscCall(PetscOptionsBoundedInt("-default_quadrature_order", "Quadrature order is one less than quadrature points per edge", "DMPlexCreateCoordinateSpace", qorder, &qorder, NULL, 0));
212     PetscOptionsEnd();
213     if (degree == PETSC_DECIDE) fe = NULL;
214     else {
215       DMPolytopeType ct;
216       PetscInt       cStart, cEnd, ctTmp;
217 
218       PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
219       // Want to match cell types
220       if (cEnd > cStart) PetscCall(DMPlexGetCellType(dm, cStart, &ct));
221       else ct = DM_POLYTOPE_UNKNOWN;
222       ctTmp = (PetscInt)ct;
223       PetscCall(MPIU_Allreduce(MPI_IN_PLACE, &ctTmp, 1, MPIU_INT, MPI_MIN, PetscObjectComm((PetscObject)dm)));
224       ct = (DMPolytopeType)ctTmp;
225       // Work around current bug
226       if (ct == DM_POLYTOPE_SEG_PRISM_TENSOR || ct == DM_POLYTOPE_TRI_PRISM_TENSOR || ct == DM_POLYTOPE_QUAD_PRISM_TENSOR) fe = NULL;
227       else PetscCall(PetscFECreateLagrangeByCell(PETSC_COMM_SELF, dim, dE, ct, degree, qorder, &fe));
228     }
229     if (fe) PetscCall(DMProjectCoordinates(dm, fe));
230     PetscCall(PetscFEDestroy(&fe));
231   }
232   mesh->coordFunc = coordFunc;
233   PetscFunctionReturn(PETSC_SUCCESS);
234 }
235 
236 /*@
237   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
238 
239   Collective
240 
241   Input Parameters:
242 + comm            - The communicator for the `DM` object
243 . dim             - The spatial dimension
244 . simplex         - Flag for simplicial cells, otherwise they are tensor product cells
245 . interpolate     - Flag to create intermediate mesh pieces (edges, faces)
246 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
247 
248   Output Parameter:
249 . newdm - The `DM` object
250 
251   Level: beginner
252 
253 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetType()`, `DMCreate()`
254 @*/
255 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscReal refinementLimit, DM *newdm)
256 {
257   DM          dm;
258   PetscMPIInt rank;
259 
260   PetscFunctionBegin;
261   PetscCall(DMCreate(comm, &dm));
262   PetscCall(DMSetType(dm, DMPLEX));
263   PetscCall(DMSetDimension(dm, dim));
264   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
265   PetscCallMPI(MPI_Comm_rank(comm, &rank));
266   switch (dim) {
267   case 2:
268     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "triangular"));
269     else PetscCall(PetscObjectSetName((PetscObject)dm, "quadrilateral"));
270     break;
271   case 3:
272     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "tetrahedral"));
273     else PetscCall(PetscObjectSetName((PetscObject)dm, "hexahedral"));
274     break;
275   default:
276     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
277   }
278   if (rank) {
279     PetscInt numPoints[2] = {0, 0};
280     PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL));
281   } else {
282     switch (dim) {
283     case 2:
284       if (simplex) {
285         PetscInt    numPoints[2]        = {4, 2};
286         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
287         PetscInt    cones[6]            = {2, 3, 4, 5, 4, 3};
288         PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
289         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
290 
291         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
292       } else {
293         PetscInt    numPoints[2]        = {6, 2};
294         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
295         PetscInt    cones[8]            = {2, 3, 4, 5, 3, 6, 7, 4};
296         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
297         PetscScalar vertexCoords[12]    = {-1.0, -0.5, 0.0, -0.5, 0.0, 0.5, -1.0, 0.5, 1.0, -0.5, 1.0, 0.5};
298 
299         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
300       }
301       break;
302     case 3:
303       if (simplex) {
304         PetscInt    numPoints[2]        = {5, 2};
305         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
306         PetscInt    cones[8]            = {4, 3, 5, 2, 5, 3, 4, 6};
307         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
308         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0};
309 
310         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
311       } else {
312         PetscInt    numPoints[2]         = {12, 2};
313         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
314         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9, 5, 4, 10, 11, 7, 12, 13, 8};
315         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
316         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5, -1.0, 0.5, -0.5, 0.0, 0.5, -0.5, 0.0, -0.5, -0.5, -1.0, -0.5, 0.5, 0.0, -0.5, 0.5, 0.0, 0.5, 0.5, -1.0, 0.5, 0.5, 1.0, 0.5, -0.5, 1.0, -0.5, -0.5, 1.0, -0.5, 0.5, 1.0, 0.5, 0.5};
317 
318         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
319       }
320       break;
321     default:
322       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
323     }
324   }
325   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
326   *newdm = dm;
327   if (refinementLimit > 0.0) {
328     DM          rdm;
329     const char *name;
330 
331     PetscCall(DMPlexSetRefinementUniform(*newdm, PETSC_FALSE));
332     PetscCall(DMPlexSetRefinementLimit(*newdm, refinementLimit));
333     PetscCall(DMRefine(*newdm, comm, &rdm));
334     PetscCall(PetscObjectGetName((PetscObject)*newdm, &name));
335     PetscCall(PetscObjectSetName((PetscObject)rdm, name));
336     PetscCall(DMDestroy(newdm));
337     *newdm = rdm;
338   }
339   if (interpolate) {
340     DM idm;
341 
342     PetscCall(DMPlexInterpolate(*newdm, &idm));
343     PetscCall(DMDestroy(newdm));
344     *newdm = idm;
345   }
346   PetscFunctionReturn(PETSC_SUCCESS);
347 }
348 
349 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
350 {
351   const PetscInt numVertices    = 2;
352   PetscInt       markerRight    = 1;
353   PetscInt       markerLeft     = 1;
354   PetscBool      markerSeparate = PETSC_FALSE;
355   Vec            coordinates;
356   PetscSection   coordSection;
357   PetscScalar   *coords;
358   PetscInt       coordSize;
359   PetscMPIInt    rank;
360   PetscInt       cdim = 1, v;
361 
362   PetscFunctionBegin;
363   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
364   if (markerSeparate) {
365     markerRight = 2;
366     markerLeft  = 1;
367   }
368   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
369   if (rank == 0) {
370     PetscCall(DMPlexSetChart(dm, 0, numVertices));
371     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
372     PetscCall(DMSetLabelValue(dm, "marker", 0, markerLeft));
373     PetscCall(DMSetLabelValue(dm, "marker", 1, markerRight));
374   }
375   PetscCall(DMPlexSymmetrize(dm));
376   PetscCall(DMPlexStratify(dm));
377   /* Build coordinates */
378   PetscCall(DMSetCoordinateDim(dm, cdim));
379   PetscCall(DMGetCoordinateSection(dm, &coordSection));
380   PetscCall(PetscSectionSetNumFields(coordSection, 1));
381   PetscCall(PetscSectionSetChart(coordSection, 0, numVertices));
382   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
383   for (v = 0; v < numVertices; ++v) {
384     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
385     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
386   }
387   PetscCall(PetscSectionSetUp(coordSection));
388   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
389   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
390   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
391   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
392   PetscCall(VecSetBlockSize(coordinates, cdim));
393   PetscCall(VecSetType(coordinates, VECSTANDARD));
394   PetscCall(VecGetArray(coordinates, &coords));
395   coords[0] = lower[0];
396   coords[1] = upper[0];
397   PetscCall(VecRestoreArray(coordinates, &coords));
398   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
399   PetscCall(VecDestroy(&coordinates));
400   PetscFunctionReturn(PETSC_SUCCESS);
401 }
402 
403 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
404 {
405   const PetscInt numVertices    = (edges[0] + 1) * (edges[1] + 1);
406   const PetscInt numEdges       = edges[0] * (edges[1] + 1) + (edges[0] + 1) * edges[1];
407   PetscInt       markerTop      = 1;
408   PetscInt       markerBottom   = 1;
409   PetscInt       markerRight    = 1;
410   PetscInt       markerLeft     = 1;
411   PetscBool      markerSeparate = PETSC_FALSE;
412   Vec            coordinates;
413   PetscSection   coordSection;
414   PetscScalar   *coords;
415   PetscInt       coordSize;
416   PetscMPIInt    rank;
417   PetscInt       v, vx, vy;
418 
419   PetscFunctionBegin;
420   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
421   if (markerSeparate) {
422     markerTop    = 3;
423     markerBottom = 1;
424     markerRight  = 2;
425     markerLeft   = 4;
426   }
427   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
428   if (rank == 0) {
429     PetscInt e, ex, ey;
430 
431     PetscCall(DMPlexSetChart(dm, 0, numEdges + numVertices));
432     for (e = 0; e < numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
433     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
434     for (vx = 0; vx <= edges[0]; vx++) {
435       for (ey = 0; ey < edges[1]; ey++) {
436         PetscInt edge   = vx * edges[1] + ey + edges[0] * (edges[1] + 1);
437         PetscInt vertex = ey * (edges[0] + 1) + vx + numEdges;
438         PetscInt cone[2];
439 
440         cone[0] = vertex;
441         cone[1] = vertex + edges[0] + 1;
442         PetscCall(DMPlexSetCone(dm, edge, cone));
443         if (vx == edges[0]) {
444           PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
445           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
446           if (ey == edges[1] - 1) {
447             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
448             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerRight));
449           }
450         } else if (vx == 0) {
451           PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
452           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
453           if (ey == edges[1] - 1) {
454             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
455             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft));
456           }
457         }
458       }
459     }
460     for (vy = 0; vy <= edges[1]; vy++) {
461       for (ex = 0; ex < edges[0]; ex++) {
462         PetscInt edge   = vy * edges[0] + ex;
463         PetscInt vertex = vy * (edges[0] + 1) + ex + numEdges;
464         PetscInt cone[2];
465 
466         cone[0] = vertex;
467         cone[1] = vertex + 1;
468         PetscCall(DMPlexSetCone(dm, edge, cone));
469         if (vy == edges[1]) {
470           PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
471           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
472           if (ex == edges[0] - 1) {
473             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
474             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerTop));
475           }
476         } else if (vy == 0) {
477           PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
478           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
479           if (ex == edges[0] - 1) {
480             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
481             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom));
482           }
483         }
484       }
485     }
486   }
487   PetscCall(DMPlexSymmetrize(dm));
488   PetscCall(DMPlexStratify(dm));
489   /* Build coordinates */
490   PetscCall(DMSetCoordinateDim(dm, 2));
491   PetscCall(DMGetCoordinateSection(dm, &coordSection));
492   PetscCall(PetscSectionSetNumFields(coordSection, 1));
493   PetscCall(PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices));
494   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 2));
495   for (v = numEdges; v < numEdges + numVertices; ++v) {
496     PetscCall(PetscSectionSetDof(coordSection, v, 2));
497     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 2));
498   }
499   PetscCall(PetscSectionSetUp(coordSection));
500   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
501   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
502   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
503   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
504   PetscCall(VecSetBlockSize(coordinates, 2));
505   PetscCall(VecSetType(coordinates, VECSTANDARD));
506   PetscCall(VecGetArray(coordinates, &coords));
507   for (vy = 0; vy <= edges[1]; ++vy) {
508     for (vx = 0; vx <= edges[0]; ++vx) {
509       coords[(vy * (edges[0] + 1) + vx) * 2 + 0] = lower[0] + ((upper[0] - lower[0]) / edges[0]) * vx;
510       coords[(vy * (edges[0] + 1) + vx) * 2 + 1] = lower[1] + ((upper[1] - lower[1]) / edges[1]) * vy;
511     }
512   }
513   PetscCall(VecRestoreArray(coordinates, &coords));
514   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
515   PetscCall(VecDestroy(&coordinates));
516   PetscFunctionReturn(PETSC_SUCCESS);
517 }
518 
519 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
520 {
521   PetscInt     vertices[3], numVertices;
522   PetscInt     numFaces       = 2 * faces[0] * faces[1] + 2 * faces[1] * faces[2] + 2 * faces[0] * faces[2];
523   PetscInt     markerTop      = 1;
524   PetscInt     markerBottom   = 1;
525   PetscInt     markerFront    = 1;
526   PetscInt     markerBack     = 1;
527   PetscInt     markerRight    = 1;
528   PetscInt     markerLeft     = 1;
529   PetscBool    markerSeparate = PETSC_FALSE;
530   Vec          coordinates;
531   PetscSection coordSection;
532   PetscScalar *coords;
533   PetscInt     coordSize;
534   PetscMPIInt  rank;
535   PetscInt     v, vx, vy, vz;
536   PetscInt     voffset, iface = 0, cone[4];
537 
538   PetscFunctionBegin;
539   PetscCheck(faces[0] >= 1 && faces[1] >= 1 && faces[2] >= 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
540   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
541   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
542   if (markerSeparate) {
543     markerBottom = 1;
544     markerTop    = 2;
545     markerFront  = 3;
546     markerBack   = 4;
547     markerRight  = 5;
548     markerLeft   = 6;
549   }
550   vertices[0] = faces[0] + 1;
551   vertices[1] = faces[1] + 1;
552   vertices[2] = faces[2] + 1;
553   numVertices = vertices[0] * vertices[1] * vertices[2];
554   if (rank == 0) {
555     PetscInt f;
556 
557     PetscCall(DMPlexSetChart(dm, 0, numFaces + numVertices));
558     for (f = 0; f < numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
559     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
560 
561     /* Side 0 (Top) */
562     for (vy = 0; vy < faces[1]; vy++) {
563       for (vx = 0; vx < faces[0]; vx++) {
564         voffset = numFaces + vertices[0] * vertices[1] * (vertices[2] - 1) + vy * vertices[0] + vx;
565         cone[0] = voffset;
566         cone[1] = voffset + 1;
567         cone[2] = voffset + vertices[0] + 1;
568         cone[3] = voffset + vertices[0];
569         PetscCall(DMPlexSetCone(dm, iface, cone));
570         PetscCall(DMSetLabelValue(dm, "marker", iface, markerTop));
571         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerTop));
572         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerTop));
573         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerTop));
574         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerTop));
575         iface++;
576       }
577     }
578 
579     /* Side 1 (Bottom) */
580     for (vy = 0; vy < faces[1]; vy++) {
581       for (vx = 0; vx < faces[0]; vx++) {
582         voffset = numFaces + vy * (faces[0] + 1) + vx;
583         cone[0] = voffset + 1;
584         cone[1] = voffset;
585         cone[2] = voffset + vertices[0];
586         cone[3] = voffset + vertices[0] + 1;
587         PetscCall(DMPlexSetCone(dm, iface, cone));
588         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBottom));
589         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBottom));
590         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBottom));
591         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerBottom));
592         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerBottom));
593         iface++;
594       }
595     }
596 
597     /* Side 2 (Front) */
598     for (vz = 0; vz < faces[2]; vz++) {
599       for (vx = 0; vx < faces[0]; vx++) {
600         voffset = numFaces + vz * vertices[0] * vertices[1] + vx;
601         cone[0] = voffset;
602         cone[1] = voffset + 1;
603         cone[2] = voffset + vertices[0] * vertices[1] + 1;
604         cone[3] = voffset + vertices[0] * vertices[1];
605         PetscCall(DMPlexSetCone(dm, iface, cone));
606         PetscCall(DMSetLabelValue(dm, "marker", iface, markerFront));
607         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerFront));
608         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerFront));
609         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerFront));
610         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerFront));
611         iface++;
612       }
613     }
614 
615     /* Side 3 (Back) */
616     for (vz = 0; vz < faces[2]; vz++) {
617       for (vx = 0; vx < faces[0]; vx++) {
618         voffset = numFaces + vz * vertices[0] * vertices[1] + vertices[0] * (vertices[1] - 1) + vx;
619         cone[0] = voffset + vertices[0] * vertices[1];
620         cone[1] = voffset + vertices[0] * vertices[1] + 1;
621         cone[2] = voffset + 1;
622         cone[3] = voffset;
623         PetscCall(DMPlexSetCone(dm, iface, cone));
624         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBack));
625         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBack));
626         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBack));
627         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerBack));
628         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerBack));
629         iface++;
630       }
631     }
632 
633     /* Side 4 (Left) */
634     for (vz = 0; vz < faces[2]; vz++) {
635       for (vy = 0; vy < faces[1]; vy++) {
636         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0];
637         cone[0] = voffset;
638         cone[1] = voffset + vertices[0] * vertices[1];
639         cone[2] = voffset + vertices[0] * vertices[1] + vertices[0];
640         cone[3] = voffset + vertices[0];
641         PetscCall(DMPlexSetCone(dm, iface, cone));
642         PetscCall(DMSetLabelValue(dm, "marker", iface, markerLeft));
643         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerLeft));
644         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerLeft));
645         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[1] + 0, markerLeft));
646         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerLeft));
647         iface++;
648       }
649     }
650 
651     /* Side 5 (Right) */
652     for (vz = 0; vz < faces[2]; vz++) {
653       for (vy = 0; vy < faces[1]; vy++) {
654         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0] + faces[0];
655         cone[0] = voffset + vertices[0] * vertices[1];
656         cone[1] = voffset;
657         cone[2] = voffset + vertices[0];
658         cone[3] = voffset + vertices[0] * vertices[1] + vertices[0];
659         PetscCall(DMPlexSetCone(dm, iface, cone));
660         PetscCall(DMSetLabelValue(dm, "marker", iface, markerRight));
661         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerRight));
662         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerRight));
663         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerRight));
664         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerRight));
665         iface++;
666       }
667     }
668   }
669   PetscCall(DMPlexSymmetrize(dm));
670   PetscCall(DMPlexStratify(dm));
671   /* Build coordinates */
672   PetscCall(DMSetCoordinateDim(dm, 3));
673   PetscCall(DMGetCoordinateSection(dm, &coordSection));
674   PetscCall(PetscSectionSetNumFields(coordSection, 1));
675   PetscCall(PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices));
676   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 3));
677   for (v = numFaces; v < numFaces + numVertices; ++v) {
678     PetscCall(PetscSectionSetDof(coordSection, v, 3));
679     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 3));
680   }
681   PetscCall(PetscSectionSetUp(coordSection));
682   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
683   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
684   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
685   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
686   PetscCall(VecSetBlockSize(coordinates, 3));
687   PetscCall(VecSetType(coordinates, VECSTANDARD));
688   PetscCall(VecGetArray(coordinates, &coords));
689   for (vz = 0; vz <= faces[2]; ++vz) {
690     for (vy = 0; vy <= faces[1]; ++vy) {
691       for (vx = 0; vx <= faces[0]; ++vx) {
692         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 0] = lower[0] + ((upper[0] - lower[0]) / faces[0]) * vx;
693         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 1] = lower[1] + ((upper[1] - lower[1]) / faces[1]) * vy;
694         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 2] = lower[2] + ((upper[2] - lower[2]) / faces[2]) * vz;
695       }
696     }
697   }
698   PetscCall(VecRestoreArray(coordinates, &coords));
699   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
700   PetscCall(VecDestroy(&coordinates));
701   PetscFunctionReturn(PETSC_SUCCESS);
702 }
703 
704 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate)
705 {
706   PetscFunctionBegin;
707   PetscValidLogicalCollectiveInt(dm, dim, 2);
708   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
709   PetscCall(DMSetDimension(dm, dim - 1));
710   PetscCall(DMSetCoordinateDim(dm, dim));
711   switch (dim) {
712   case 1:
713     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(dm, lower, upper, faces));
714     break;
715   case 2:
716     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(dm, lower, upper, faces));
717     break;
718   case 3:
719     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(dm, lower, upper, faces));
720     break;
721   default:
722     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Dimension not supported: %" PetscInt_FMT, dim);
723   }
724   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
725   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
726   PetscFunctionReturn(PETSC_SUCCESS);
727 }
728 
729 /*@C
730   DMPlexCreateBoxSurfaceMesh - Creates a mesh on the surface of the tensor product of unit intervals (box) using tensor cells (hexahedra).
731 
732   Collective
733 
734   Input Parameters:
735 + comm        - The communicator for the `DM` object
736 . dim         - The spatial dimension of the box, so the resulting mesh is has dimension `dim`-1
737 . faces       - Number of faces per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
738 . lower       - The lower left corner, or `NULL` for (0, 0, 0)
739 . upper       - The upper right corner, or `NULL` for (1, 1, 1)
740 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
741 
742   Output Parameter:
743 . dm - The `DM` object
744 
745   Level: beginner
746 
747 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateBoxMesh()`, `DMPlexCreateFromFile()`, `DMSetType()`, `DMCreate()`
748 @*/
749 PetscErrorCode DMPlexCreateBoxSurfaceMesh(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate, DM *dm)
750 {
751   PetscInt  fac[3] = {1, 1, 1};
752   PetscReal low[3] = {0, 0, 0};
753   PetscReal upp[3] = {1, 1, 1};
754 
755   PetscFunctionBegin;
756   PetscCall(DMCreate(comm, dm));
757   PetscCall(DMSetType(*dm, DMPLEX));
758   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(*dm, dim, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, interpolate));
759   PetscFunctionReturn(PETSC_SUCCESS);
760 }
761 
762 static PetscErrorCode DMPlexCreateLineMesh_Internal(DM dm, PetscInt segments, PetscReal lower, PetscReal upper, DMBoundaryType bd)
763 {
764   PetscInt     i, fStart, fEnd, numCells = 0, numVerts = 0;
765   PetscInt     numPoints[2], *coneSize, *cones, *coneOrientations;
766   PetscScalar *vertexCoords;
767   PetscReal    L, maxCell;
768   PetscBool    markerSeparate = PETSC_FALSE;
769   PetscInt     markerLeft = 1, faceMarkerLeft = 1;
770   PetscInt     markerRight = 1, faceMarkerRight = 2;
771   PetscBool    wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
772   PetscMPIInt  rank;
773 
774   PetscFunctionBegin;
775   PetscAssertPointer(dm, 1);
776 
777   PetscCall(DMSetDimension(dm, 1));
778   PetscCall(DMCreateLabel(dm, "marker"));
779   PetscCall(DMCreateLabel(dm, "Face Sets"));
780 
781   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
782   if (rank == 0) numCells = segments;
783   if (rank == 0) numVerts = segments + (wrap ? 0 : 1);
784 
785   numPoints[0] = numVerts;
786   numPoints[1] = numCells;
787   PetscCall(PetscMalloc4(numCells + numVerts, &coneSize, numCells * 2, &cones, numCells + numVerts, &coneOrientations, numVerts, &vertexCoords));
788   PetscCall(PetscArrayzero(coneOrientations, numCells + numVerts));
789   for (i = 0; i < numCells; ++i) coneSize[i] = 2;
790   for (i = 0; i < numVerts; ++i) coneSize[numCells + i] = 0;
791   for (i = 0; i < numCells; ++i) {
792     cones[2 * i]     = numCells + i % numVerts;
793     cones[2 * i + 1] = numCells + (i + 1) % numVerts;
794   }
795   for (i = 0; i < numVerts; ++i) vertexCoords[i] = lower + (upper - lower) * ((PetscReal)i / (PetscReal)numCells);
796   PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
797   PetscCall(PetscFree4(coneSize, cones, coneOrientations, vertexCoords));
798 
799   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
800   if (markerSeparate) {
801     markerLeft  = faceMarkerLeft;
802     markerRight = faceMarkerRight;
803   }
804   if (!wrap && rank == 0) {
805     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
806     PetscCall(DMSetLabelValue(dm, "marker", fStart, markerLeft));
807     PetscCall(DMSetLabelValue(dm, "marker", fEnd - 1, markerRight));
808     PetscCall(DMSetLabelValue(dm, "Face Sets", fStart, faceMarkerLeft));
809     PetscCall(DMSetLabelValue(dm, "Face Sets", fEnd - 1, faceMarkerRight));
810   }
811   if (wrap) {
812     L       = upper - lower;
813     maxCell = (PetscReal)1.1 * (L / (PetscReal)PetscMax(1, segments));
814     PetscCall(DMSetPeriodicity(dm, &maxCell, &lower, &L));
815   }
816   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
817   PetscFunctionReturn(PETSC_SUCCESS);
818 }
819 
820 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
821 {
822   DM      boundary, vol;
823   DMLabel bdlabel;
824 
825   PetscFunctionBegin;
826   PetscAssertPointer(dm, 1);
827   for (PetscInt i = 0; i < dim; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
828   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &boundary));
829   PetscCall(DMSetType(boundary, DMPLEX));
830   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(boundary, dim, faces, lower, upper, PETSC_FALSE));
831   PetscCall(DMPlexGenerate(boundary, NULL, interpolate, &vol));
832   PetscCall(DMGetLabel(vol, "marker", &bdlabel));
833   if (bdlabel) PetscCall(DMPlexLabelComplete(vol, bdlabel));
834   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_FALSE, vol));
835   PetscCall(DMPlexReplace_Internal(dm, &vol));
836   PetscCall(DMDestroy(&boundary));
837   PetscFunctionReturn(PETSC_SUCCESS);
838 }
839 
840 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
841 {
842   DMLabel     cutLabel  = NULL;
843   PetscInt    markerTop = 1, faceMarkerTop = 1;
844   PetscInt    markerBottom = 1, faceMarkerBottom = 1;
845   PetscInt    markerFront = 1, faceMarkerFront = 1;
846   PetscInt    markerBack = 1, faceMarkerBack = 1;
847   PetscInt    markerRight = 1, faceMarkerRight = 1;
848   PetscInt    markerLeft = 1, faceMarkerLeft = 1;
849   PetscInt    dim;
850   PetscBool   markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
851   PetscMPIInt rank;
852 
853   PetscFunctionBegin;
854   PetscCall(DMGetDimension(dm, &dim));
855   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
856   PetscCall(DMCreateLabel(dm, "marker"));
857   PetscCall(DMCreateLabel(dm, "Face Sets"));
858   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
859   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST || bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST || bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
860     if (cutMarker) {
861       PetscCall(DMCreateLabel(dm, "periodic_cut"));
862       PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
863     }
864   }
865   switch (dim) {
866   case 2:
867     faceMarkerTop    = 3;
868     faceMarkerBottom = 1;
869     faceMarkerRight  = 2;
870     faceMarkerLeft   = 4;
871     break;
872   case 3:
873     faceMarkerBottom = 1;
874     faceMarkerTop    = 2;
875     faceMarkerFront  = 3;
876     faceMarkerBack   = 4;
877     faceMarkerRight  = 5;
878     faceMarkerLeft   = 6;
879     break;
880   default:
881     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
882   }
883   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
884   if (markerSeparate) {
885     markerBottom = faceMarkerBottom;
886     markerTop    = faceMarkerTop;
887     markerFront  = faceMarkerFront;
888     markerBack   = faceMarkerBack;
889     markerRight  = faceMarkerRight;
890     markerLeft   = faceMarkerLeft;
891   }
892   {
893     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
894     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
895     const PetscInt numZEdges    = rank == 0 ? edges[2] : 0;
896     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0] + 1) : 0;
897     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1] + 1) : 0;
898     const PetscInt numZVertices = rank == 0 ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2] + 1) : 0;
899     const PetscInt numCells     = numXEdges * numYEdges * numZEdges;
900     const PetscInt numXFaces    = numYEdges * numZEdges;
901     const PetscInt numYFaces    = numXEdges * numZEdges;
902     const PetscInt numZFaces    = numXEdges * numYEdges;
903     const PetscInt numTotXFaces = numXVertices * numXFaces;
904     const PetscInt numTotYFaces = numYVertices * numYFaces;
905     const PetscInt numTotZFaces = numZVertices * numZFaces;
906     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
907     const PetscInt numTotXEdges = numXEdges * numYVertices * numZVertices;
908     const PetscInt numTotYEdges = numYEdges * numXVertices * numZVertices;
909     const PetscInt numTotZEdges = numZEdges * numXVertices * numYVertices;
910     const PetscInt numVertices  = numXVertices * numYVertices * numZVertices;
911     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
912     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
913     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
914     const PetscInt firstYFace   = firstXFace + numTotXFaces;
915     const PetscInt firstZFace   = firstYFace + numTotYFaces;
916     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
917     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
918     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
919     Vec            coordinates;
920     PetscSection   coordSection;
921     PetscScalar   *coords;
922     PetscInt       coordSize;
923     PetscInt       v, vx, vy, vz;
924     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
925 
926     PetscCall(DMPlexSetChart(dm, 0, numCells + numFaces + numEdges + numVertices));
927     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
928     for (f = firstXFace; f < firstXFace + numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
929     for (e = firstXEdge; e < firstXEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
930     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
931     /* Build cells */
932     for (fz = 0; fz < numZEdges; ++fz) {
933       for (fy = 0; fy < numYEdges; ++fy) {
934         for (fx = 0; fx < numXEdges; ++fx) {
935           PetscInt cell  = (fz * numYEdges + fy) * numXEdges + fx;
936           PetscInt faceB = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
937           PetscInt faceT = firstZFace + (fy * numXEdges + fx) * numZVertices + ((fz + 1) % numZVertices);
938           PetscInt faceF = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
939           PetscInt faceK = firstYFace + (fz * numXEdges + fx) * numYVertices + ((fy + 1) % numYVertices);
940           PetscInt faceL = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
941           PetscInt faceR = firstXFace + (fz * numYEdges + fy) * numXVertices + ((fx + 1) % numXVertices);
942           /* B,  T,  F,  K,  R,  L */
943           PetscInt ornt[6] = {-2, 0, 0, -3, 0, -2}; /* ??? */
944           PetscInt cone[6];
945 
946           /* no boundary twisting in 3D */
947           cone[0] = faceB;
948           cone[1] = faceT;
949           cone[2] = faceF;
950           cone[3] = faceK;
951           cone[4] = faceR;
952           cone[5] = faceL;
953           PetscCall(DMPlexSetCone(dm, cell, cone));
954           PetscCall(DMPlexSetConeOrientation(dm, cell, ornt));
955           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
956           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
957           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
958         }
959       }
960     }
961     /* Build x faces */
962     for (fz = 0; fz < numZEdges; ++fz) {
963       for (fy = 0; fy < numYEdges; ++fy) {
964         for (fx = 0; fx < numXVertices; ++fx) {
965           PetscInt face    = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
966           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
967           PetscInt edgeR   = firstZEdge + (((fy + 1) % numYVertices) * numXVertices + fx) * numZEdges + fz;
968           PetscInt edgeB   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
969           PetscInt edgeT   = firstYEdge + (((fz + 1) % numZVertices) * numXVertices + fx) * numYEdges + fy;
970           PetscInt ornt[4] = {0, 0, -1, -1};
971           PetscInt cone[4];
972 
973           if (dim == 3) {
974             /* markers */
975             if (bdX != DM_BOUNDARY_PERIODIC) {
976               if (fx == numXVertices - 1) {
977                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight));
978                 PetscCall(DMSetLabelValue(dm, "marker", face, markerRight));
979               } else if (fx == 0) {
980                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft));
981                 PetscCall(DMSetLabelValue(dm, "marker", face, markerLeft));
982               }
983             }
984           }
985           cone[0] = edgeB;
986           cone[1] = edgeR;
987           cone[2] = edgeT;
988           cone[3] = edgeL;
989           PetscCall(DMPlexSetCone(dm, face, cone));
990           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
991         }
992       }
993     }
994     /* Build y faces */
995     for (fz = 0; fz < numZEdges; ++fz) {
996       for (fx = 0; fx < numXEdges; ++fx) {
997         for (fy = 0; fy < numYVertices; ++fy) {
998           PetscInt face    = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
999           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
1000           PetscInt edgeR   = firstZEdge + (fy * numXVertices + ((fx + 1) % numXVertices)) * numZEdges + fz;
1001           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1002           PetscInt edgeT   = firstXEdge + (((fz + 1) % numZVertices) * numYVertices + fy) * numXEdges + fx;
1003           PetscInt ornt[4] = {0, 0, -1, -1};
1004           PetscInt cone[4];
1005 
1006           if (dim == 3) {
1007             /* markers */
1008             if (bdY != DM_BOUNDARY_PERIODIC) {
1009               if (fy == numYVertices - 1) {
1010                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack));
1011                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBack));
1012               } else if (fy == 0) {
1013                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront));
1014                 PetscCall(DMSetLabelValue(dm, "marker", face, markerFront));
1015               }
1016             }
1017           }
1018           cone[0] = edgeB;
1019           cone[1] = edgeR;
1020           cone[2] = edgeT;
1021           cone[3] = edgeL;
1022           PetscCall(DMPlexSetCone(dm, face, cone));
1023           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1024         }
1025       }
1026     }
1027     /* Build z faces */
1028     for (fy = 0; fy < numYEdges; ++fy) {
1029       for (fx = 0; fx < numXEdges; ++fx) {
1030         for (fz = 0; fz < numZVertices; fz++) {
1031           PetscInt face    = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
1032           PetscInt edgeL   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
1033           PetscInt edgeR   = firstYEdge + (fz * numXVertices + ((fx + 1) % numXVertices)) * numYEdges + fy;
1034           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1035           PetscInt edgeT   = firstXEdge + (fz * numYVertices + ((fy + 1) % numYVertices)) * numXEdges + fx;
1036           PetscInt ornt[4] = {0, 0, -1, -1};
1037           PetscInt cone[4];
1038 
1039           if (dim == 2) {
1040             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges - 1) {
1041               edgeR += numYEdges - 1 - 2 * fy;
1042               ornt[1] = -1;
1043             }
1044             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges - 1) {
1045               edgeT += numXEdges - 1 - 2 * fx;
1046               ornt[2] = 0;
1047             }
1048             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1049             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1050           } else {
1051             /* markers */
1052             if (bdZ != DM_BOUNDARY_PERIODIC) {
1053               if (fz == numZVertices - 1) {
1054                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop));
1055                 PetscCall(DMSetLabelValue(dm, "marker", face, markerTop));
1056               } else if (fz == 0) {
1057                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom));
1058                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBottom));
1059               }
1060             }
1061           }
1062           cone[0] = edgeB;
1063           cone[1] = edgeR;
1064           cone[2] = edgeT;
1065           cone[3] = edgeL;
1066           PetscCall(DMPlexSetCone(dm, face, cone));
1067           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1068         }
1069       }
1070     }
1071     /* Build Z edges*/
1072     for (vy = 0; vy < numYVertices; vy++) {
1073       for (vx = 0; vx < numXVertices; vx++) {
1074         for (ez = 0; ez < numZEdges; ez++) {
1075           const PetscInt edge    = firstZEdge + (vy * numXVertices + vx) * numZEdges + ez;
1076           const PetscInt vertexB = firstVertex + (ez * numYVertices + vy) * numXVertices + vx;
1077           const PetscInt vertexT = firstVertex + (((ez + 1) % numZVertices) * numYVertices + vy) * numXVertices + vx;
1078           PetscInt       cone[2];
1079 
1080           cone[0] = vertexB;
1081           cone[1] = vertexT;
1082           PetscCall(DMPlexSetCone(dm, edge, cone));
1083           if (dim == 3) {
1084             if (bdX != DM_BOUNDARY_PERIODIC) {
1085               if (vx == numXVertices - 1) {
1086                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1087                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1088                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1089               } else if (vx == 0) {
1090                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1091                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1092                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1093               }
1094             }
1095             if (bdY != DM_BOUNDARY_PERIODIC) {
1096               if (vy == numYVertices - 1) {
1097                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1098                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1099                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1100               } else if (vy == 0) {
1101                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1102                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1103                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1104               }
1105             }
1106           }
1107         }
1108       }
1109     }
1110     /* Build Y edges*/
1111     for (vz = 0; vz < numZVertices; vz++) {
1112       for (vx = 0; vx < numXVertices; vx++) {
1113         for (ey = 0; ey < numYEdges; ey++) {
1114           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges - 1) ? (numXVertices - vx - 1) : (vz * numYVertices + ((ey + 1) % numYVertices)) * numXVertices + vx;
1115           const PetscInt edge    = firstYEdge + (vz * numXVertices + vx) * numYEdges + ey;
1116           const PetscInt vertexF = firstVertex + (vz * numYVertices + ey) * numXVertices + vx;
1117           const PetscInt vertexK = firstVertex + nextv;
1118           PetscInt       cone[2];
1119 
1120           cone[0] = vertexF;
1121           cone[1] = vertexK;
1122           PetscCall(DMPlexSetCone(dm, edge, cone));
1123           if (dim == 2) {
1124             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1125               if (vx == numXVertices - 1) {
1126                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight));
1127                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1128                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1129                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1130               } else if (vx == 0) {
1131                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft));
1132                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1133                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1134                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1135               }
1136             } else {
1137               if (vx == 0 && cutLabel) {
1138                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1139                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1140                 if (ey == numYEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1141               }
1142             }
1143           } else {
1144             if (bdX != DM_BOUNDARY_PERIODIC) {
1145               if (vx == numXVertices - 1) {
1146                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1147                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1148                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1149               } else if (vx == 0) {
1150                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1151                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1152                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1153               }
1154             }
1155             if (bdZ != DM_BOUNDARY_PERIODIC) {
1156               if (vz == numZVertices - 1) {
1157                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1158                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1159                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1160               } else if (vz == 0) {
1161                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1162                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1163                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1164               }
1165             }
1166           }
1167         }
1168       }
1169     }
1170     /* Build X edges*/
1171     for (vz = 0; vz < numZVertices; vz++) {
1172       for (vy = 0; vy < numYVertices; vy++) {
1173         for (ex = 0; ex < numXEdges; ex++) {
1174           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges - 1) ? (numYVertices - vy - 1) * numXVertices : (vz * numYVertices + vy) * numXVertices + (ex + 1) % numXVertices;
1175           const PetscInt edge    = firstXEdge + (vz * numYVertices + vy) * numXEdges + ex;
1176           const PetscInt vertexL = firstVertex + (vz * numYVertices + vy) * numXVertices + ex;
1177           const PetscInt vertexR = firstVertex + nextv;
1178           PetscInt       cone[2];
1179 
1180           cone[0] = vertexL;
1181           cone[1] = vertexR;
1182           PetscCall(DMPlexSetCone(dm, edge, cone));
1183           if (dim == 2) {
1184             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1185               if (vy == numYVertices - 1) {
1186                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop));
1187                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1188                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1189                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1190               } else if (vy == 0) {
1191                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom));
1192                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1193                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1194                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1195               }
1196             } else {
1197               if (vy == 0 && cutLabel) {
1198                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1199                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1200                 if (ex == numXEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1201               }
1202             }
1203           } else {
1204             if (bdY != DM_BOUNDARY_PERIODIC) {
1205               if (vy == numYVertices - 1) {
1206                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1207                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1208                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1209               } else if (vy == 0) {
1210                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1211                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1212                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1213               }
1214             }
1215             if (bdZ != DM_BOUNDARY_PERIODIC) {
1216               if (vz == numZVertices - 1) {
1217                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1218                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1219                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1220               } else if (vz == 0) {
1221                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1222                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1223                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1224               }
1225             }
1226           }
1227         }
1228       }
1229     }
1230     PetscCall(DMPlexSymmetrize(dm));
1231     PetscCall(DMPlexStratify(dm));
1232     /* Build coordinates */
1233     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1234     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1235     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1236     PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVertices));
1237     for (v = firstVertex; v < firstVertex + numVertices; ++v) {
1238       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1239       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1240     }
1241     PetscCall(PetscSectionSetUp(coordSection));
1242     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1243     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1244     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1245     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1246     PetscCall(VecSetBlockSize(coordinates, dim));
1247     PetscCall(VecSetType(coordinates, VECSTANDARD));
1248     PetscCall(VecGetArray(coordinates, &coords));
1249     for (vz = 0; vz < numZVertices; ++vz) {
1250       for (vy = 0; vy < numYVertices; ++vy) {
1251         for (vx = 0; vx < numXVertices; ++vx) {
1252           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * vx;
1253           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * vy;
1254           if (dim == 3) coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 2] = lower[2] + ((upper[2] - lower[2]) / numZEdges) * vz;
1255         }
1256       }
1257     }
1258     PetscCall(VecRestoreArray(coordinates, &coords));
1259     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1260     PetscCall(VecDestroy(&coordinates));
1261   }
1262   PetscFunctionReturn(PETSC_SUCCESS);
1263 }
1264 
1265 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1266 {
1267   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1268   PetscInt       fac[3] = {0, 0, 0}, d;
1269 
1270   PetscFunctionBegin;
1271   PetscAssertPointer(dm, 1);
1272   PetscValidLogicalCollectiveInt(dm, dim, 2);
1273   PetscCall(DMSetDimension(dm, dim));
1274   for (d = 0; d < dim; ++d) {
1275     fac[d] = faces[d];
1276     bdt[d] = periodicity[d];
1277   }
1278   PetscCall(DMPlexCreateCubeMesh_Internal(dm, lower, upper, fac, bdt[0], bdt[1], bdt[2]));
1279   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST || periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST || (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
1280     PetscReal L[3]       = {-1., -1., 0.};
1281     PetscReal maxCell[3] = {-1., -1., 0.};
1282 
1283     for (d = 0; d < dim; ++d) {
1284       if (periodicity[d] != DM_BOUNDARY_NONE) {
1285         L[d]       = upper[d] - lower[d];
1286         maxCell[d] = 1.1 * (L[d] / PetscMax(1, faces[d]));
1287       }
1288     }
1289     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1290   }
1291   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
1292   PetscFunctionReturn(PETSC_SUCCESS);
1293 }
1294 
1295 static PetscErrorCode DMPlexCreateBoxMesh_Internal(DM dm, DMPlexShape shape, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1296 {
1297   PetscFunctionBegin;
1298   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
1299   if (shape == DM_SHAPE_ZBOX) PetscCall(DMPlexCreateBoxMesh_Tensor_SFC_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1300   else if (dim == 1) PetscCall(DMPlexCreateLineMesh_Internal(dm, faces[0], lower[0], upper[0], periodicity[0]));
1301   else if (simplex) PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1302   else PetscCall(DMPlexCreateBoxMesh_Tensor_Internal(dm, dim, faces, lower, upper, periodicity));
1303   if (!interpolate && dim > 1 && !simplex) {
1304     DM udm;
1305 
1306     PetscCall(DMPlexUninterpolate(dm, &udm));
1307     PetscCall(DMPlexCopyCoordinates(dm, udm));
1308     PetscCall(DMPlexReplace_Internal(dm, &udm));
1309   }
1310   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
1311   PetscFunctionReturn(PETSC_SUCCESS);
1312 }
1313 
1314 /*@C
1315   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
1316 
1317   Collective
1318 
1319   Input Parameters:
1320 + comm        - The communicator for the `DM` object
1321 . dim         - The spatial dimension
1322 . simplex     - `PETSC_TRUE` for simplices, `PETSC_FALSE` for tensor cells
1323 . faces       - Number of faces per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1324 . lower       - The lower left corner, or `NULL` for (0, 0, 0)
1325 . upper       - The upper right corner, or `NULL` for (1, 1, 1)
1326 . periodicity - The boundary type for the X,Y,Z direction, or `NULL` for `DM_BOUNDARY_NONE`
1327 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1328 
1329   Output Parameter:
1330 . dm - The `DM` object
1331 
1332   Level: beginner
1333 
1334   Note:
1335   To customize this mesh using options, use
1336 .vb
1337   DMCreate(comm, &dm);
1338   DMSetType(dm, DMPLEX);
1339   DMSetFromOptions(dm);
1340 .ve
1341   and use the options in `DMSetFromOptions()`.
1342 
1343   Here is the numbering returned for 2 faces in each direction for tensor cells:
1344 .vb
1345  10---17---11---18----12
1346   |         |         |
1347   |         |         |
1348  20    2   22    3    24
1349   |         |         |
1350   |         |         |
1351   7---15----8---16----9
1352   |         |         |
1353   |         |         |
1354  19    0   21    1   23
1355   |         |         |
1356   |         |         |
1357   4---13----5---14----6
1358 .ve
1359   and for simplicial cells
1360 .vb
1361  14----8---15----9----16
1362   |\     5  |\      7 |
1363   | \       | \       |
1364  13   2    14    3    15
1365   | 4   \   | 6   \   |
1366   |       \ |       \ |
1367  11----6---12----7----13
1368   |\        |\        |
1369   | \    1  | \     3 |
1370  10   0    11    1    12
1371   | 0   \   | 2   \   |
1372   |       \ |       \ |
1373   8----4----9----5----10
1374 .ve
1375 
1376 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
1377 @*/
1378 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, DM *dm)
1379 {
1380   PetscInt       fac[3] = {1, 1, 1};
1381   PetscReal      low[3] = {0, 0, 0};
1382   PetscReal      upp[3] = {1, 1, 1};
1383   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1384 
1385   PetscFunctionBegin;
1386   PetscCall(DMCreate(comm, dm));
1387   PetscCall(DMSetType(*dm, DMPLEX));
1388   PetscCall(DMPlexCreateBoxMesh_Internal(*dm, DM_SHAPE_BOX, dim, simplex, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt, interpolate));
1389   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1390   PetscFunctionReturn(PETSC_SUCCESS);
1391 }
1392 
1393 static PetscErrorCode DMPlexCreateWedgeBoxMesh_Internal(DM dm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1394 {
1395   DM       bdm, vol;
1396   PetscInt i;
1397 
1398   PetscFunctionBegin;
1399   // TODO Now we can support periodicity
1400   for (i = 0; i < 3; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity not yet supported");
1401   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &bdm));
1402   PetscCall(DMSetType(bdm, DMPLEX));
1403   PetscCall(DMSetDimension(bdm, 2));
1404   PetscCall(PetscLogEventBegin(DMPLEX_Generate, bdm, 0, 0, 0));
1405   PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(bdm, 2, faces, lower, upper, periodicity, PETSC_TRUE));
1406   PetscCall(DMPlexExtrude(bdm, faces[2], upper[2] - lower[2], PETSC_TRUE, PETSC_FALSE, PETSC_FALSE, NULL, NULL, &vol));
1407   PetscCall(PetscLogEventEnd(DMPLEX_Generate, bdm, 0, 0, 0));
1408   PetscCall(DMDestroy(&bdm));
1409   PetscCall(DMPlexReplace_Internal(dm, &vol));
1410   if (lower[2] != 0.0) {
1411     Vec          v;
1412     PetscScalar *x;
1413     PetscInt     cDim, n;
1414 
1415     PetscCall(DMGetCoordinatesLocal(dm, &v));
1416     PetscCall(VecGetBlockSize(v, &cDim));
1417     PetscCall(VecGetLocalSize(v, &n));
1418     PetscCall(VecGetArray(v, &x));
1419     x += cDim;
1420     for (i = 0; i < n; i += cDim) x[i] += lower[2];
1421     PetscCall(VecRestoreArray(v, &x));
1422     PetscCall(DMSetCoordinatesLocal(dm, v));
1423   }
1424   PetscFunctionReturn(PETSC_SUCCESS);
1425 }
1426 
1427 /*@
1428   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tesselating the (x,y) plane and extruding in the third direction using wedge cells.
1429 
1430   Collective
1431 
1432   Input Parameters:
1433 + comm        - The communicator for the `DM` object
1434 . faces       - Number of faces per dimension, or `NULL` for (1, 1, 1)
1435 . lower       - The lower left corner, or `NULL` for (0, 0, 0)
1436 . upper       - The upper right corner, or `NULL` for (1, 1, 1)
1437 . periodicity - The boundary type for the X,Y,Z direction, or `NULL` for `DM_BOUNDARY_NONE`
1438 . orderHeight - If `PETSC_TRUE`, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1439 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1440 
1441   Output Parameter:
1442 . dm - The `DM` object
1443 
1444   Level: beginner
1445 
1446 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateWedgeCylinderMesh()`, `DMExtrude()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1447 @*/
1448 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool orderHeight, PetscBool interpolate, DM *dm)
1449 {
1450   PetscInt       fac[3] = {1, 1, 1};
1451   PetscReal      low[3] = {0, 0, 0};
1452   PetscReal      upp[3] = {1, 1, 1};
1453   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1454 
1455   PetscFunctionBegin;
1456   PetscCall(DMCreate(comm, dm));
1457   PetscCall(DMSetType(*dm, DMPLEX));
1458   PetscCall(DMPlexCreateWedgeBoxMesh_Internal(*dm, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt));
1459   if (!interpolate) {
1460     DM udm;
1461 
1462     PetscCall(DMPlexUninterpolate(*dm, &udm));
1463     PetscCall(DMPlexReplace_Internal(*dm, &udm));
1464   }
1465   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1466   PetscFunctionReturn(PETSC_SUCCESS);
1467 }
1468 
1469 /*
1470   DMPlexTensorPointLexicographic_Private - Returns all tuples of size 'len' with nonnegative integers that are all less than or equal to 'max' for that dimension.
1471 
1472   Input Parameters:
1473 + len - The length of the tuple
1474 . max - The maximum for each dimension, so values are in [0, max)
1475 - tup - A tuple of length len+1: tup[len] > 0 indicates a stopping condition
1476 
1477   Output Parameter:
1478 . tup - A tuple of `len` integers whose entries are at most `max`
1479 
1480   Level: developer
1481 
1482   Note:
1483   Ordering is lexicographic with lowest index as least significant in ordering.
1484   e.g. for len == 2 and max == 2, this will return, in order, {0,0}, {1,0}, {2,0}, {0,1}, {1,1}, {2,1}, {0,2}, {1,2}, {2,2}.
1485 
1486 .seealso: PetscDualSpaceTensorPointLexicographic_Internal(), PetscDualSpaceLatticePointLexicographic_Internal()
1487 */
1488 static PetscErrorCode DMPlexTensorPointLexicographic_Private(PetscInt len, const PetscInt max[], PetscInt tup[])
1489 {
1490   PetscInt i;
1491 
1492   PetscFunctionBegin;
1493   for (i = 0; i < len; ++i) {
1494     if (tup[i] < max[i] - 1) {
1495       break;
1496     } else {
1497       tup[i] = 0;
1498     }
1499   }
1500   if (i == len) tup[i - 1] = max[i - 1];
1501   else ++tup[i];
1502   PetscFunctionReturn(PETSC_SUCCESS);
1503 }
1504 
1505 static PetscInt TupleToIndex_Private(PetscInt len, const PetscInt max[], const PetscInt tup[])
1506 {
1507   PetscInt i, idx = tup[len - 1];
1508 
1509   for (i = len - 2; i >= 0; --i) {
1510     idx *= max[i];
1511     idx += tup[i];
1512   }
1513   return idx;
1514 }
1515 
1516 static PetscErrorCode DestroyExtent_Private(void *extent)
1517 {
1518   return PetscFree(extent);
1519 }
1520 
1521 static PetscErrorCode DMPlexCreateHypercubicMesh_Internal(DM dm, PetscInt dim, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], const DMBoundaryType bd[])
1522 {
1523   Vec          coordinates;
1524   PetscSection coordSection;
1525   DMLabel      cutLabel    = NULL;
1526   PetscBool    cutMarker   = PETSC_FALSE;
1527   PetscBool    periodic    = PETSC_FALSE;
1528   PetscInt     numCells    = 1, c;
1529   PetscInt     numVertices = 1, v;
1530   PetscScalar *coords;
1531   PetscInt    *vertices, *vert, *vtmp, *supp, cone[2];
1532   PetscInt     d, e, cell = 0, coordSize;
1533   PetscMPIInt  rank;
1534 
1535   PetscFunctionBegin;
1536   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1537   PetscCall(DMSetDimension(dm, dim));
1538   PetscCall(PetscCalloc4(dim, &vertices, dim, &vert, dim, &vtmp, 2 * dim, &supp));
1539   PetscCall(DMCreateLabel(dm, "marker"));
1540   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
1541   for (d = 0; d < dim; ++d) periodic = (periodic || bd[d] == DM_BOUNDARY_PERIODIC) ? PETSC_TRUE : PETSC_FALSE;
1542   if (periodic && cutMarker) {
1543     PetscCall(DMCreateLabel(dm, "periodic_cut"));
1544     PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
1545   }
1546   for (d = 0; d < dim; ++d) PetscCheck(bd[d] == DM_BOUNDARY_PERIODIC, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Hypercubic mesh must be periodic now");
1547   for (d = 0; d < dim; ++d) {
1548     vertices[d] = edges[d];
1549     numVertices *= vertices[d];
1550   }
1551   numCells = numVertices * dim;
1552   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1553   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, 2));
1554   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetSupportSize(dm, v, 2 * dim));
1555   /* TODO Loop over boundary and reset support sizes */
1556   PetscCall(DMSetUp(dm)); /* Allocate space for cones and supports */
1557   /* Build cell cones and vertex supports */
1558   PetscCall(DMCreateLabel(dm, "celltype"));
1559   while (vert[dim - 1] < vertices[dim - 1]) {
1560     const PetscInt vertex = TupleToIndex_Private(dim, vertices, vert) + numCells;
1561     PetscInt       s      = 0;
1562 
1563     PetscCall(PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT ":", vertex));
1564     for (d = 0; d < dim; ++d) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, vert[d]));
1565     PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
1566     PetscCall(DMPlexSetCellType(dm, vertex, DM_POLYTOPE_POINT));
1567     for (d = 0; d < dim; ++d) {
1568       for (e = 0; e < dim; ++e) vtmp[e] = vert[e];
1569       vtmp[d] = (vert[d] + 1) % vertices[d];
1570       cone[0] = vertex;
1571       cone[1] = TupleToIndex_Private(dim, vertices, vtmp) + numCells;
1572       PetscCall(PetscPrintf(PETSC_COMM_SELF, "  Vertex %" PetscInt_FMT ":", cone[1]));
1573       for (e = 0; e < dim; ++e) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, vtmp[e]));
1574       PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
1575       PetscCall(DMPlexSetCone(dm, cell, cone));
1576       PetscCall(DMPlexSetCellType(dm, cell, DM_POLYTOPE_SEGMENT));
1577       PetscCall(PetscPrintf(PETSC_COMM_SELF, "  Edge %" PetscInt_FMT " (%" PetscInt_FMT " %" PetscInt_FMT ")\n", cell, cone[0], cone[1]));
1578       ++cell;
1579     }
1580     for (d = 0; d < dim; ++d) {
1581       for (e = 0; e < dim; ++e) vtmp[e] = vert[e];
1582       vtmp[d]   = (vert[d] + vertices[d] - 1) % vertices[d];
1583       supp[s++] = TupleToIndex_Private(dim, vertices, vtmp) * dim + d;
1584       supp[s++] = (vertex - numCells) * dim + d;
1585       PetscCall(DMPlexSetSupport(dm, vertex, supp));
1586     }
1587     PetscCall(DMPlexTensorPointLexicographic_Private(dim, vertices, vert));
1588   }
1589   PetscCall(DMPlexStratify(dm));
1590   /* Build coordinates */
1591   PetscCall(DMGetCoordinateSection(dm, &coordSection));
1592   PetscCall(PetscSectionSetNumFields(coordSection, 1));
1593   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1594   PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1595   for (v = numCells; v < numCells + numVertices; ++v) {
1596     PetscCall(PetscSectionSetDof(coordSection, v, dim));
1597     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1598   }
1599   PetscCall(PetscSectionSetUp(coordSection));
1600   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1601   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1602   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1603   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1604   PetscCall(VecSetBlockSize(coordinates, dim));
1605   PetscCall(VecSetType(coordinates, VECSTANDARD));
1606   PetscCall(VecGetArray(coordinates, &coords));
1607   for (d = 0; d < dim; ++d) vert[d] = 0;
1608   while (vert[dim - 1] < vertices[dim - 1]) {
1609     const PetscInt vertex = TupleToIndex_Private(dim, vertices, vert);
1610 
1611     for (d = 0; d < dim; ++d) coords[vertex * dim + d] = lower[d] + ((upper[d] - lower[d]) / vertices[d]) * vert[d];
1612     PetscCall(PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT ":", vertex));
1613     for (d = 0; d < dim; ++d) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, vert[d]));
1614     for (d = 0; d < dim; ++d) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %g", (double)PetscRealPart(coords[vertex * dim + d])));
1615     PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
1616     PetscCall(DMPlexTensorPointLexicographic_Private(dim, vertices, vert));
1617   }
1618   PetscCall(VecRestoreArray(coordinates, &coords));
1619   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1620   PetscCall(VecDestroy(&coordinates));
1621   PetscCall(PetscFree4(vertices, vert, vtmp, supp));
1622   //PetscCall(DMSetPeriodicity(dm, NULL, lower, upper));
1623   // Attach the extent
1624   {
1625     PetscContainer c;
1626     PetscInt      *extent;
1627 
1628     PetscCall(PetscMalloc1(dim, &extent));
1629     for (PetscInt d = 0; d < dim; ++d) extent[d] = edges[d];
1630     PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &c));
1631     PetscCall(PetscContainerSetUserDestroy(c, DestroyExtent_Private));
1632     PetscCall(PetscContainerSetPointer(c, extent));
1633     PetscCall(PetscObjectCompose((PetscObject)dm, "_extent", (PetscObject)c));
1634     PetscCall(PetscContainerDestroy(&c));
1635   }
1636   PetscFunctionReturn(PETSC_SUCCESS);
1637 }
1638 
1639 /*@C
1640   DMPlexCreateHypercubicMesh - Creates a periodic mesh on the tensor product of unit intervals using only vertices and edges.
1641 
1642   Collective
1643 
1644   Input Parameters:
1645 + comm  - The communicator for the DM object
1646 . dim   - The spatial dimension
1647 . edges - Number of edges per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1648 . lower - The lower left corner, or `NULL` for (0, 0, 0)
1649 - upper - The upper right corner, or `NULL` for (1, 1, 1)
1650 
1651   Output Parameter:
1652 . dm - The DM object
1653 
1654   Level: beginner
1655 
1656   Note:
1657   If you want to customize this mesh using options, you just need to
1658 .vb
1659   DMCreate(comm, &dm);
1660   DMSetType(dm, DMPLEX);
1661   DMSetFromOptions(dm);
1662 .ve
1663   and use the options on the `DMSetFromOptions()` page.
1664 
1665   The vertices are numbered is lexicographic order, and the dim edges exiting a vertex in the positive orthant are number consecutively,
1666 .vb
1667  18--0-19--2-20--4-18
1668   |     |     |     |
1669  13    15    17    13
1670   |     |     |     |
1671  24-12-25-14-26-16-24
1672   |     |     |     |
1673   7     9    11     7
1674   |     |     |     |
1675  21--6-22--8-23-10-21
1676   |     |     |     |
1677   1     3     5     1
1678   |     |     |     |
1679  18--0-19--2-20--4-18
1680 .ve
1681 
1682 .seealso: `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
1683 @*/
1684 PetscErrorCode DMPlexCreateHypercubicMesh(MPI_Comm comm, PetscInt dim, const PetscInt edges[], const PetscReal lower[], const PetscReal upper[], DM *dm)
1685 {
1686   PetscInt       *edg;
1687   PetscReal      *low, *upp;
1688   DMBoundaryType *bdt;
1689   PetscInt        d;
1690 
1691   PetscFunctionBegin;
1692   PetscCall(DMCreate(comm, dm));
1693   PetscCall(DMSetType(*dm, DMPLEX));
1694   PetscCall(PetscMalloc4(dim, &edg, dim, &low, dim, &upp, dim, &bdt));
1695   for (d = 0; d < dim; ++d) {
1696     edg[d] = edges ? edges[d] : 1;
1697     low[d] = lower ? lower[d] : 0.;
1698     upp[d] = upper ? upper[d] : 1.;
1699     bdt[d] = DM_BOUNDARY_PERIODIC;
1700   }
1701   PetscCall(DMPlexCreateHypercubicMesh_Internal(*dm, dim, low, upp, edg, bdt));
1702   PetscCall(PetscFree4(edg, low, upp, bdt));
1703   PetscFunctionReturn(PETSC_SUCCESS);
1704 }
1705 
1706 /*@C
1707   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all `DM` options in the database.
1708 
1709   Logically Collective
1710 
1711   Input Parameters:
1712 + dm     - the `DM` context
1713 - prefix - the prefix to prepend to all option names
1714 
1715   Level: advanced
1716 
1717   Note:
1718   A hyphen (-) must NOT be given at the beginning of the prefix name.
1719   The first character of all runtime options is AUTOMATICALLY the hyphen.
1720 
1721 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `SNESSetFromOptions()`
1722 @*/
1723 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
1724 {
1725   DM_Plex *mesh = (DM_Plex *)dm->data;
1726 
1727   PetscFunctionBegin;
1728   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1729   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, prefix));
1730   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)mesh->partitioner, prefix));
1731   PetscFunctionReturn(PETSC_SUCCESS);
1732 }
1733 
1734 /* Remap geometry to cylinder
1735    TODO: This only works for a single refinement, then it is broken
1736 
1737      Interior square: Linear interpolation is correct
1738      The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
1739      such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
1740 
1741        phi     = arctan(y/x)
1742        d_close = sqrt(1/8 + 1/4 sin^2(phi))
1743        d_far   = sqrt(1/2 + sin^2(phi))
1744 
1745      so we remap them using
1746 
1747        x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
1748        y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
1749 
1750      If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
1751 */
1752 static void snapToCylinder(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
1753 {
1754   const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
1755   const PetscReal ds2 = 0.5 * dis;
1756 
1757   if ((PetscAbsScalar(u[0]) <= ds2) && (PetscAbsScalar(u[1]) <= ds2)) {
1758     f0[0] = u[0];
1759     f0[1] = u[1];
1760   } else {
1761     PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
1762 
1763     x    = PetscRealPart(u[0]);
1764     y    = PetscRealPart(u[1]);
1765     phi  = PetscAtan2Real(y, x);
1766     sinp = PetscSinReal(phi);
1767     cosp = PetscCosReal(phi);
1768     if ((PetscAbsReal(phi) > PETSC_PI / 4.0) && (PetscAbsReal(phi) < 3.0 * PETSC_PI / 4.0)) {
1769       dc = PetscAbsReal(ds2 / sinp);
1770       df = PetscAbsReal(dis / sinp);
1771       xc = ds2 * x / PetscAbsReal(y);
1772       yc = ds2 * PetscSignReal(y);
1773     } else {
1774       dc = PetscAbsReal(ds2 / cosp);
1775       df = PetscAbsReal(dis / cosp);
1776       xc = ds2 * PetscSignReal(x);
1777       yc = ds2 * y / PetscAbsReal(x);
1778     }
1779     f0[0] = xc + (u[0] - xc) * (1.0 - dc) / (df - dc);
1780     f0[1] = yc + (u[1] - yc) * (1.0 - dc) / (df - dc);
1781   }
1782   f0[2] = u[2];
1783 }
1784 
1785 static PetscErrorCode DMPlexCreateHexCylinderMesh_Internal(DM dm, DMBoundaryType periodicZ)
1786 {
1787   const PetscInt dim = 3;
1788   PetscInt       numCells, numVertices;
1789   PetscMPIInt    rank;
1790 
1791   PetscFunctionBegin;
1792   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
1793   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1794   PetscCall(DMSetDimension(dm, dim));
1795   /* Create topology */
1796   {
1797     PetscInt cone[8], c;
1798 
1799     numCells    = rank == 0 ? 5 : 0;
1800     numVertices = rank == 0 ? 16 : 0;
1801     if (periodicZ == DM_BOUNDARY_PERIODIC) {
1802       numCells *= 3;
1803       numVertices = rank == 0 ? 24 : 0;
1804     }
1805     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1806     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 8));
1807     PetscCall(DMSetUp(dm));
1808     if (rank == 0) {
1809       if (periodicZ == DM_BOUNDARY_PERIODIC) {
1810         cone[0] = 15;
1811         cone[1] = 18;
1812         cone[2] = 17;
1813         cone[3] = 16;
1814         cone[4] = 31;
1815         cone[5] = 32;
1816         cone[6] = 33;
1817         cone[7] = 34;
1818         PetscCall(DMPlexSetCone(dm, 0, cone));
1819         cone[0] = 16;
1820         cone[1] = 17;
1821         cone[2] = 24;
1822         cone[3] = 23;
1823         cone[4] = 32;
1824         cone[5] = 36;
1825         cone[6] = 37;
1826         cone[7] = 33; /* 22 25 26 21 */
1827         PetscCall(DMPlexSetCone(dm, 1, cone));
1828         cone[0] = 18;
1829         cone[1] = 27;
1830         cone[2] = 24;
1831         cone[3] = 17;
1832         cone[4] = 34;
1833         cone[5] = 33;
1834         cone[6] = 37;
1835         cone[7] = 38;
1836         PetscCall(DMPlexSetCone(dm, 2, cone));
1837         cone[0] = 29;
1838         cone[1] = 27;
1839         cone[2] = 18;
1840         cone[3] = 15;
1841         cone[4] = 35;
1842         cone[5] = 31;
1843         cone[6] = 34;
1844         cone[7] = 38;
1845         PetscCall(DMPlexSetCone(dm, 3, cone));
1846         cone[0] = 29;
1847         cone[1] = 15;
1848         cone[2] = 16;
1849         cone[3] = 23;
1850         cone[4] = 35;
1851         cone[5] = 36;
1852         cone[6] = 32;
1853         cone[7] = 31;
1854         PetscCall(DMPlexSetCone(dm, 4, cone));
1855 
1856         cone[0] = 31;
1857         cone[1] = 34;
1858         cone[2] = 33;
1859         cone[3] = 32;
1860         cone[4] = 19;
1861         cone[5] = 22;
1862         cone[6] = 21;
1863         cone[7] = 20;
1864         PetscCall(DMPlexSetCone(dm, 5, cone));
1865         cone[0] = 32;
1866         cone[1] = 33;
1867         cone[2] = 37;
1868         cone[3] = 36;
1869         cone[4] = 22;
1870         cone[5] = 25;
1871         cone[6] = 26;
1872         cone[7] = 21;
1873         PetscCall(DMPlexSetCone(dm, 6, cone));
1874         cone[0] = 34;
1875         cone[1] = 38;
1876         cone[2] = 37;
1877         cone[3] = 33;
1878         cone[4] = 20;
1879         cone[5] = 21;
1880         cone[6] = 26;
1881         cone[7] = 28;
1882         PetscCall(DMPlexSetCone(dm, 7, cone));
1883         cone[0] = 35;
1884         cone[1] = 38;
1885         cone[2] = 34;
1886         cone[3] = 31;
1887         cone[4] = 30;
1888         cone[5] = 19;
1889         cone[6] = 20;
1890         cone[7] = 28;
1891         PetscCall(DMPlexSetCone(dm, 8, cone));
1892         cone[0] = 35;
1893         cone[1] = 31;
1894         cone[2] = 32;
1895         cone[3] = 36;
1896         cone[4] = 30;
1897         cone[5] = 25;
1898         cone[6] = 22;
1899         cone[7] = 19;
1900         PetscCall(DMPlexSetCone(dm, 9, cone));
1901 
1902         cone[0] = 19;
1903         cone[1] = 20;
1904         cone[2] = 21;
1905         cone[3] = 22;
1906         cone[4] = 15;
1907         cone[5] = 16;
1908         cone[6] = 17;
1909         cone[7] = 18;
1910         PetscCall(DMPlexSetCone(dm, 10, cone));
1911         cone[0] = 22;
1912         cone[1] = 21;
1913         cone[2] = 26;
1914         cone[3] = 25;
1915         cone[4] = 16;
1916         cone[5] = 23;
1917         cone[6] = 24;
1918         cone[7] = 17;
1919         PetscCall(DMPlexSetCone(dm, 11, cone));
1920         cone[0] = 20;
1921         cone[1] = 28;
1922         cone[2] = 26;
1923         cone[3] = 21;
1924         cone[4] = 18;
1925         cone[5] = 17;
1926         cone[6] = 24;
1927         cone[7] = 27;
1928         PetscCall(DMPlexSetCone(dm, 12, cone));
1929         cone[0] = 30;
1930         cone[1] = 28;
1931         cone[2] = 20;
1932         cone[3] = 19;
1933         cone[4] = 29;
1934         cone[5] = 15;
1935         cone[6] = 18;
1936         cone[7] = 27;
1937         PetscCall(DMPlexSetCone(dm, 13, cone));
1938         cone[0] = 30;
1939         cone[1] = 19;
1940         cone[2] = 22;
1941         cone[3] = 25;
1942         cone[4] = 29;
1943         cone[5] = 23;
1944         cone[6] = 16;
1945         cone[7] = 15;
1946         PetscCall(DMPlexSetCone(dm, 14, cone));
1947       } else {
1948         cone[0] = 5;
1949         cone[1] = 8;
1950         cone[2] = 7;
1951         cone[3] = 6;
1952         cone[4] = 9;
1953         cone[5] = 12;
1954         cone[6] = 11;
1955         cone[7] = 10;
1956         PetscCall(DMPlexSetCone(dm, 0, cone));
1957         cone[0] = 6;
1958         cone[1] = 7;
1959         cone[2] = 14;
1960         cone[3] = 13;
1961         cone[4] = 12;
1962         cone[5] = 15;
1963         cone[6] = 16;
1964         cone[7] = 11;
1965         PetscCall(DMPlexSetCone(dm, 1, cone));
1966         cone[0] = 8;
1967         cone[1] = 17;
1968         cone[2] = 14;
1969         cone[3] = 7;
1970         cone[4] = 10;
1971         cone[5] = 11;
1972         cone[6] = 16;
1973         cone[7] = 18;
1974         PetscCall(DMPlexSetCone(dm, 2, cone));
1975         cone[0] = 19;
1976         cone[1] = 17;
1977         cone[2] = 8;
1978         cone[3] = 5;
1979         cone[4] = 20;
1980         cone[5] = 9;
1981         cone[6] = 10;
1982         cone[7] = 18;
1983         PetscCall(DMPlexSetCone(dm, 3, cone));
1984         cone[0] = 19;
1985         cone[1] = 5;
1986         cone[2] = 6;
1987         cone[3] = 13;
1988         cone[4] = 20;
1989         cone[5] = 15;
1990         cone[6] = 12;
1991         cone[7] = 9;
1992         PetscCall(DMPlexSetCone(dm, 4, cone));
1993       }
1994     }
1995     PetscCall(DMPlexSymmetrize(dm));
1996     PetscCall(DMPlexStratify(dm));
1997   }
1998   /* Create cube geometry */
1999   {
2000     Vec             coordinates;
2001     PetscSection    coordSection;
2002     PetscScalar    *coords;
2003     PetscInt        coordSize, v;
2004     const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
2005     const PetscReal ds2 = dis / 2.0;
2006 
2007     /* Build coordinates */
2008     PetscCall(DMGetCoordinateSection(dm, &coordSection));
2009     PetscCall(PetscSectionSetNumFields(coordSection, 1));
2010     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
2011     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
2012     for (v = numCells; v < numCells + numVertices; ++v) {
2013       PetscCall(PetscSectionSetDof(coordSection, v, dim));
2014       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
2015     }
2016     PetscCall(PetscSectionSetUp(coordSection));
2017     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2018     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2019     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2020     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2021     PetscCall(VecSetBlockSize(coordinates, dim));
2022     PetscCall(VecSetType(coordinates, VECSTANDARD));
2023     PetscCall(VecGetArray(coordinates, &coords));
2024     if (rank == 0) {
2025       coords[0 * dim + 0]  = -ds2;
2026       coords[0 * dim + 1]  = -ds2;
2027       coords[0 * dim + 2]  = 0.0;
2028       coords[1 * dim + 0]  = ds2;
2029       coords[1 * dim + 1]  = -ds2;
2030       coords[1 * dim + 2]  = 0.0;
2031       coords[2 * dim + 0]  = ds2;
2032       coords[2 * dim + 1]  = ds2;
2033       coords[2 * dim + 2]  = 0.0;
2034       coords[3 * dim + 0]  = -ds2;
2035       coords[3 * dim + 1]  = ds2;
2036       coords[3 * dim + 2]  = 0.0;
2037       coords[4 * dim + 0]  = -ds2;
2038       coords[4 * dim + 1]  = -ds2;
2039       coords[4 * dim + 2]  = 1.0;
2040       coords[5 * dim + 0]  = -ds2;
2041       coords[5 * dim + 1]  = ds2;
2042       coords[5 * dim + 2]  = 1.0;
2043       coords[6 * dim + 0]  = ds2;
2044       coords[6 * dim + 1]  = ds2;
2045       coords[6 * dim + 2]  = 1.0;
2046       coords[7 * dim + 0]  = ds2;
2047       coords[7 * dim + 1]  = -ds2;
2048       coords[7 * dim + 2]  = 1.0;
2049       coords[8 * dim + 0]  = dis;
2050       coords[8 * dim + 1]  = -dis;
2051       coords[8 * dim + 2]  = 0.0;
2052       coords[9 * dim + 0]  = dis;
2053       coords[9 * dim + 1]  = dis;
2054       coords[9 * dim + 2]  = 0.0;
2055       coords[10 * dim + 0] = dis;
2056       coords[10 * dim + 1] = -dis;
2057       coords[10 * dim + 2] = 1.0;
2058       coords[11 * dim + 0] = dis;
2059       coords[11 * dim + 1] = dis;
2060       coords[11 * dim + 2] = 1.0;
2061       coords[12 * dim + 0] = -dis;
2062       coords[12 * dim + 1] = dis;
2063       coords[12 * dim + 2] = 0.0;
2064       coords[13 * dim + 0] = -dis;
2065       coords[13 * dim + 1] = dis;
2066       coords[13 * dim + 2] = 1.0;
2067       coords[14 * dim + 0] = -dis;
2068       coords[14 * dim + 1] = -dis;
2069       coords[14 * dim + 2] = 0.0;
2070       coords[15 * dim + 0] = -dis;
2071       coords[15 * dim + 1] = -dis;
2072       coords[15 * dim + 2] = 1.0;
2073       if (periodicZ == DM_BOUNDARY_PERIODIC) {
2074         /* 15 31 19 */ coords[16 * dim + 0] = -ds2;
2075         coords[16 * dim + 1]                = -ds2;
2076         coords[16 * dim + 2]                = 0.5;
2077         /* 16 32 22 */ coords[17 * dim + 0] = ds2;
2078         coords[17 * dim + 1]                = -ds2;
2079         coords[17 * dim + 2]                = 0.5;
2080         /* 17 33 21 */ coords[18 * dim + 0] = ds2;
2081         coords[18 * dim + 1]                = ds2;
2082         coords[18 * dim + 2]                = 0.5;
2083         /* 18 34 20 */ coords[19 * dim + 0] = -ds2;
2084         coords[19 * dim + 1]                = ds2;
2085         coords[19 * dim + 2]                = 0.5;
2086         /* 29 35 30 */ coords[20 * dim + 0] = -dis;
2087         coords[20 * dim + 1]                = -dis;
2088         coords[20 * dim + 2]                = 0.5;
2089         /* 23 36 25 */ coords[21 * dim + 0] = dis;
2090         coords[21 * dim + 1]                = -dis;
2091         coords[21 * dim + 2]                = 0.5;
2092         /* 24 37 26 */ coords[22 * dim + 0] = dis;
2093         coords[22 * dim + 1]                = dis;
2094         coords[22 * dim + 2]                = 0.5;
2095         /* 27 38 28 */ coords[23 * dim + 0] = -dis;
2096         coords[23 * dim + 1]                = dis;
2097         coords[23 * dim + 2]                = 0.5;
2098       }
2099     }
2100     PetscCall(VecRestoreArray(coordinates, &coords));
2101     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2102     PetscCall(VecDestroy(&coordinates));
2103   }
2104   /* Create periodicity */
2105   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
2106     PetscReal L[3]       = {-1., -1., 0.};
2107     PetscReal maxCell[3] = {-1., -1., 0.};
2108     PetscReal lower[3]   = {0.0, 0.0, 0.0};
2109     PetscReal upper[3]   = {1.0, 1.0, 1.5};
2110     PetscInt  numZCells  = 3;
2111 
2112     L[2]       = upper[2] - lower[2];
2113     maxCell[2] = 1.1 * (L[2] / numZCells);
2114     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
2115   }
2116   {
2117     DM          cdm;
2118     PetscDS     cds;
2119     PetscScalar c[2] = {1.0, 1.0};
2120 
2121     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToCylinder));
2122     PetscCall(DMGetCoordinateDM(dm, &cdm));
2123     PetscCall(DMGetDS(cdm, &cds));
2124     PetscCall(PetscDSSetConstants(cds, 2, c));
2125   }
2126   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
2127 
2128   /* Wait for coordinate creation before doing in-place modification */
2129   PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2130   PetscFunctionReturn(PETSC_SUCCESS);
2131 }
2132 
2133 /*@
2134   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
2135 
2136   Collective
2137 
2138   Input Parameters:
2139 + comm      - The communicator for the `DM` object
2140 - periodicZ - The boundary type for the Z direction
2141 
2142   Output Parameter:
2143 . dm - The `DM` object
2144 
2145   Level: beginner
2146 
2147   Note:
2148 
2149   Here is the output numbering looking from the bottom of the cylinder:
2150 .vb
2151        17-----14
2152         |     |
2153         |  2  |
2154         |     |
2155  17-----8-----7-----14
2156   |     |     |     |
2157   |  3  |  0  |  1  |
2158   |     |     |     |
2159  19-----5-----6-----13
2160         |     |
2161         |  4  |
2162         |     |
2163        19-----13
2164 
2165  and up through the top
2166 
2167        18-----16
2168         |     |
2169         |  2  |
2170         |     |
2171  18----10----11-----16
2172   |     |     |     |
2173   |  3  |  0  |  1  |
2174   |     |     |     |
2175  20-----9----12-----15
2176         |     |
2177         |  4  |
2178         |     |
2179        20-----15
2180 .ve
2181 
2182 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2183 @*/
2184 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, DMBoundaryType periodicZ, DM *dm)
2185 {
2186   PetscFunctionBegin;
2187   PetscAssertPointer(dm, 3);
2188   PetscCall(DMCreate(comm, dm));
2189   PetscCall(DMSetType(*dm, DMPLEX));
2190   PetscCall(DMPlexCreateHexCylinderMesh_Internal(*dm, periodicZ));
2191   PetscFunctionReturn(PETSC_SUCCESS);
2192 }
2193 
2194 static PetscErrorCode DMPlexCreateWedgeCylinderMesh_Internal(DM dm, PetscInt n, PetscBool interpolate)
2195 {
2196   const PetscInt dim = 3;
2197   PetscInt       numCells, numVertices, v;
2198   PetscMPIInt    rank;
2199 
2200   PetscFunctionBegin;
2201   PetscCheck(n >= 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %" PetscInt_FMT " cannot be negative", n);
2202   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
2203   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2204   PetscCall(DMSetDimension(dm, dim));
2205   /* Must create the celltype label here so that we do not automatically try to compute the types */
2206   PetscCall(DMCreateLabel(dm, "celltype"));
2207   /* Create topology */
2208   {
2209     PetscInt cone[6], c;
2210 
2211     numCells    = rank == 0 ? n : 0;
2212     numVertices = rank == 0 ? 2 * (n + 1) : 0;
2213     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
2214     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
2215     PetscCall(DMSetUp(dm));
2216     for (c = 0; c < numCells; c++) {
2217       cone[0] = c + n * 1;
2218       cone[1] = (c + 1) % n + n * 1;
2219       cone[2] = 0 + 3 * n;
2220       cone[3] = c + n * 2;
2221       cone[4] = (c + 1) % n + n * 2;
2222       cone[5] = 1 + 3 * n;
2223       PetscCall(DMPlexSetCone(dm, c, cone));
2224       PetscCall(DMPlexSetCellType(dm, c, DM_POLYTOPE_TRI_PRISM_TENSOR));
2225     }
2226     PetscCall(DMPlexSymmetrize(dm));
2227     PetscCall(DMPlexStratify(dm));
2228   }
2229   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetCellType(dm, v, DM_POLYTOPE_POINT));
2230   /* Create cylinder geometry */
2231   {
2232     Vec          coordinates;
2233     PetscSection coordSection;
2234     PetscScalar *coords;
2235     PetscInt     coordSize, c;
2236 
2237     /* Build coordinates */
2238     PetscCall(DMGetCoordinateSection(dm, &coordSection));
2239     PetscCall(PetscSectionSetNumFields(coordSection, 1));
2240     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
2241     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
2242     for (v = numCells; v < numCells + numVertices; ++v) {
2243       PetscCall(PetscSectionSetDof(coordSection, v, dim));
2244       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
2245     }
2246     PetscCall(PetscSectionSetUp(coordSection));
2247     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2248     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2249     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2250     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2251     PetscCall(VecSetBlockSize(coordinates, dim));
2252     PetscCall(VecSetType(coordinates, VECSTANDARD));
2253     PetscCall(VecGetArray(coordinates, &coords));
2254     for (c = 0; c < numCells; c++) {
2255       coords[(c + 0 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
2256       coords[(c + 0 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
2257       coords[(c + 0 * n) * dim + 2] = 1.0;
2258       coords[(c + 1 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
2259       coords[(c + 1 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
2260       coords[(c + 1 * n) * dim + 2] = 0.0;
2261     }
2262     if (rank == 0) {
2263       coords[(2 * n + 0) * dim + 0] = 0.0;
2264       coords[(2 * n + 0) * dim + 1] = 0.0;
2265       coords[(2 * n + 0) * dim + 2] = 1.0;
2266       coords[(2 * n + 1) * dim + 0] = 0.0;
2267       coords[(2 * n + 1) * dim + 1] = 0.0;
2268       coords[(2 * n + 1) * dim + 2] = 0.0;
2269     }
2270     PetscCall(VecRestoreArray(coordinates, &coords));
2271     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2272     PetscCall(VecDestroy(&coordinates));
2273   }
2274   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
2275   /* Interpolate */
2276   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2277   PetscFunctionReturn(PETSC_SUCCESS);
2278 }
2279 
2280 /*@
2281   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
2282 
2283   Collective
2284 
2285   Input Parameters:
2286 + comm        - The communicator for the `DM` object
2287 . n           - The number of wedges around the origin
2288 - interpolate - Create edges and faces
2289 
2290   Output Parameter:
2291 . dm - The `DM` object
2292 
2293   Level: beginner
2294 
2295 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2296 @*/
2297 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
2298 {
2299   PetscFunctionBegin;
2300   PetscAssertPointer(dm, 4);
2301   PetscCall(DMCreate(comm, dm));
2302   PetscCall(DMSetType(*dm, DMPLEX));
2303   PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(*dm, n, interpolate));
2304   PetscFunctionReturn(PETSC_SUCCESS);
2305 }
2306 
2307 static inline PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2308 {
2309   PetscReal prod = 0.0;
2310   PetscInt  i;
2311   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
2312   return PetscSqrtReal(prod);
2313 }
2314 static inline PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2315 {
2316   PetscReal prod = 0.0;
2317   PetscInt  i;
2318   for (i = 0; i < dim; ++i) prod += x[i] * y[i];
2319   return prod;
2320 }
2321 
2322 /* The first constant is the sphere radius */
2323 static void snapToSphere(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
2324 {
2325   PetscReal r     = PetscRealPart(constants[0]);
2326   PetscReal norm2 = 0.0, fac;
2327   PetscInt  n     = uOff[1] - uOff[0], d;
2328 
2329   for (d = 0; d < n; ++d) norm2 += PetscSqr(PetscRealPart(u[d]));
2330   fac = r / PetscSqrtReal(norm2);
2331   for (d = 0; d < n; ++d) f0[d] = u[d] * fac;
2332 }
2333 
2334 static PetscErrorCode DMPlexCreateSphereMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, PetscReal R)
2335 {
2336   const PetscInt embedDim = dim + 1;
2337   PetscSection   coordSection;
2338   Vec            coordinates;
2339   PetscScalar   *coords;
2340   PetscReal     *coordsIn;
2341   PetscInt       numCells, numEdges, numVerts = 0, firstVertex = 0, v, firstEdge, coordSize, d, c, e;
2342   PetscMPIInt    rank;
2343 
2344   PetscFunctionBegin;
2345   PetscValidLogicalCollectiveBool(dm, simplex, 3);
2346   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
2347   PetscCall(DMSetDimension(dm, dim));
2348   PetscCall(DMSetCoordinateDim(dm, dim + 1));
2349   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2350   switch (dim) {
2351   case 2:
2352     if (simplex) {
2353       const PetscReal radius    = PetscSqrtReal(1 + PETSC_PHI * PETSC_PHI) / (1.0 + PETSC_PHI);
2354       const PetscReal edgeLen   = 2.0 / (1.0 + PETSC_PHI) * (R / radius);
2355       const PetscInt  degree    = 5;
2356       PetscReal       vertex[3] = {0.0, 1.0 / (1.0 + PETSC_PHI), PETSC_PHI / (1.0 + PETSC_PHI)};
2357       PetscInt        s[3]      = {1, 1, 1};
2358       PetscInt        cone[3];
2359       PetscInt       *graph, p, i, j, k;
2360 
2361       vertex[0] *= R / radius;
2362       vertex[1] *= R / radius;
2363       vertex[2] *= R / radius;
2364       numCells    = rank == 0 ? 20 : 0;
2365       numVerts    = rank == 0 ? 12 : 0;
2366       firstVertex = numCells;
2367       /* Use icosahedron, which for a R-sphere has coordinates which are all cyclic permutations of
2368 
2369            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
2370 
2371          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2372          length is then given by 2/(1+\phi) = 2 * 0.38197 = 0.76393.
2373       */
2374       /* Construct vertices */
2375       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2376       if (rank == 0) {
2377         for (p = 0, i = 0; p < embedDim; ++p) {
2378           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2379             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2380               for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertex[(d + p) % embedDim];
2381               ++i;
2382             }
2383           }
2384         }
2385       }
2386       /* Construct graph */
2387       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2388       for (i = 0; i < numVerts; ++i) {
2389         for (j = 0, k = 0; j < numVerts; ++j) {
2390           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2391             graph[i * numVerts + j] = 1;
2392             ++k;
2393           }
2394         }
2395         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid icosahedron, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2396       }
2397       /* Build Topology */
2398       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2399       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2400       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2401       /* Cells */
2402       for (i = 0, c = 0; i < numVerts; ++i) {
2403         for (j = 0; j < i; ++j) {
2404           for (k = 0; k < j; ++k) {
2405             if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i]) {
2406               cone[0] = firstVertex + i;
2407               cone[1] = firstVertex + j;
2408               cone[2] = firstVertex + k;
2409               /* Check orientation */
2410               {
2411                 const PetscInt epsilon[3][3][3] = {
2412                   {{0, 0, 0},  {0, 0, 1},  {0, -1, 0}},
2413                   {{0, 0, -1}, {0, 0, 0},  {1, 0, 0} },
2414                   {{0, 1, 0},  {-1, 0, 0}, {0, 0, 0} }
2415                 };
2416                 PetscReal normal[3];
2417                 PetscInt  e, f;
2418 
2419                 for (d = 0; d < embedDim; ++d) {
2420                   normal[d] = 0.0;
2421                   for (e = 0; e < embedDim; ++e) {
2422                     for (f = 0; f < embedDim; ++f) normal[d] += epsilon[d][e][f] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]);
2423                   }
2424                 }
2425                 if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2426                   PetscInt tmp = cone[1];
2427                   cone[1]      = cone[2];
2428                   cone[2]      = tmp;
2429                 }
2430               }
2431               PetscCall(DMPlexSetCone(dm, c++, cone));
2432             }
2433           }
2434         }
2435       }
2436       PetscCall(DMPlexSymmetrize(dm));
2437       PetscCall(DMPlexStratify(dm));
2438       PetscCall(PetscFree(graph));
2439     } else {
2440       /*
2441         12-21--13
2442          |     |
2443         25  4  24
2444          |     |
2445   12-25--9-16--8-24--13
2446    |     |     |     |
2447   23  5 17  0 15  3  22
2448    |     |     |     |
2449   10-20--6-14--7-19--11
2450          |     |
2451         20  1  19
2452          |     |
2453         10-18--11
2454          |     |
2455         23  2  22
2456          |     |
2457         12-21--13
2458        */
2459       PetscInt cone[4], ornt[4];
2460 
2461       numCells    = rank == 0 ? 6 : 0;
2462       numEdges    = rank == 0 ? 12 : 0;
2463       numVerts    = rank == 0 ? 8 : 0;
2464       firstVertex = numCells;
2465       firstEdge   = numCells + numVerts;
2466       /* Build Topology */
2467       PetscCall(DMPlexSetChart(dm, 0, numCells + numEdges + numVerts));
2468       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 4));
2469       for (e = firstEdge; e < firstEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
2470       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2471       if (rank == 0) {
2472         /* Cell 0 */
2473         cone[0] = 14;
2474         cone[1] = 15;
2475         cone[2] = 16;
2476         cone[3] = 17;
2477         PetscCall(DMPlexSetCone(dm, 0, cone));
2478         ornt[0] = 0;
2479         ornt[1] = 0;
2480         ornt[2] = 0;
2481         ornt[3] = 0;
2482         PetscCall(DMPlexSetConeOrientation(dm, 0, ornt));
2483         /* Cell 1 */
2484         cone[0] = 18;
2485         cone[1] = 19;
2486         cone[2] = 14;
2487         cone[3] = 20;
2488         PetscCall(DMPlexSetCone(dm, 1, cone));
2489         ornt[0] = 0;
2490         ornt[1] = 0;
2491         ornt[2] = -1;
2492         ornt[3] = 0;
2493         PetscCall(DMPlexSetConeOrientation(dm, 1, ornt));
2494         /* Cell 2 */
2495         cone[0] = 21;
2496         cone[1] = 22;
2497         cone[2] = 18;
2498         cone[3] = 23;
2499         PetscCall(DMPlexSetCone(dm, 2, cone));
2500         ornt[0] = 0;
2501         ornt[1] = 0;
2502         ornt[2] = -1;
2503         ornt[3] = 0;
2504         PetscCall(DMPlexSetConeOrientation(dm, 2, ornt));
2505         /* Cell 3 */
2506         cone[0] = 19;
2507         cone[1] = 22;
2508         cone[2] = 24;
2509         cone[3] = 15;
2510         PetscCall(DMPlexSetCone(dm, 3, cone));
2511         ornt[0] = -1;
2512         ornt[1] = -1;
2513         ornt[2] = 0;
2514         ornt[3] = -1;
2515         PetscCall(DMPlexSetConeOrientation(dm, 3, ornt));
2516         /* Cell 4 */
2517         cone[0] = 16;
2518         cone[1] = 24;
2519         cone[2] = 21;
2520         cone[3] = 25;
2521         PetscCall(DMPlexSetCone(dm, 4, cone));
2522         ornt[0] = -1;
2523         ornt[1] = -1;
2524         ornt[2] = -1;
2525         ornt[3] = 0;
2526         PetscCall(DMPlexSetConeOrientation(dm, 4, ornt));
2527         /* Cell 5 */
2528         cone[0] = 20;
2529         cone[1] = 17;
2530         cone[2] = 25;
2531         cone[3] = 23;
2532         PetscCall(DMPlexSetCone(dm, 5, cone));
2533         ornt[0] = -1;
2534         ornt[1] = -1;
2535         ornt[2] = -1;
2536         ornt[3] = -1;
2537         PetscCall(DMPlexSetConeOrientation(dm, 5, ornt));
2538         /* Edges */
2539         cone[0] = 6;
2540         cone[1] = 7;
2541         PetscCall(DMPlexSetCone(dm, 14, cone));
2542         cone[0] = 7;
2543         cone[1] = 8;
2544         PetscCall(DMPlexSetCone(dm, 15, cone));
2545         cone[0] = 8;
2546         cone[1] = 9;
2547         PetscCall(DMPlexSetCone(dm, 16, cone));
2548         cone[0] = 9;
2549         cone[1] = 6;
2550         PetscCall(DMPlexSetCone(dm, 17, cone));
2551         cone[0] = 10;
2552         cone[1] = 11;
2553         PetscCall(DMPlexSetCone(dm, 18, cone));
2554         cone[0] = 11;
2555         cone[1] = 7;
2556         PetscCall(DMPlexSetCone(dm, 19, cone));
2557         cone[0] = 6;
2558         cone[1] = 10;
2559         PetscCall(DMPlexSetCone(dm, 20, cone));
2560         cone[0] = 12;
2561         cone[1] = 13;
2562         PetscCall(DMPlexSetCone(dm, 21, cone));
2563         cone[0] = 13;
2564         cone[1] = 11;
2565         PetscCall(DMPlexSetCone(dm, 22, cone));
2566         cone[0] = 10;
2567         cone[1] = 12;
2568         PetscCall(DMPlexSetCone(dm, 23, cone));
2569         cone[0] = 13;
2570         cone[1] = 8;
2571         PetscCall(DMPlexSetCone(dm, 24, cone));
2572         cone[0] = 12;
2573         cone[1] = 9;
2574         PetscCall(DMPlexSetCone(dm, 25, cone));
2575       }
2576       PetscCall(DMPlexSymmetrize(dm));
2577       PetscCall(DMPlexStratify(dm));
2578       /* Build coordinates */
2579       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2580       if (rank == 0) {
2581         coordsIn[0 * embedDim + 0] = -R;
2582         coordsIn[0 * embedDim + 1] = R;
2583         coordsIn[0 * embedDim + 2] = -R;
2584         coordsIn[1 * embedDim + 0] = R;
2585         coordsIn[1 * embedDim + 1] = R;
2586         coordsIn[1 * embedDim + 2] = -R;
2587         coordsIn[2 * embedDim + 0] = R;
2588         coordsIn[2 * embedDim + 1] = -R;
2589         coordsIn[2 * embedDim + 2] = -R;
2590         coordsIn[3 * embedDim + 0] = -R;
2591         coordsIn[3 * embedDim + 1] = -R;
2592         coordsIn[3 * embedDim + 2] = -R;
2593         coordsIn[4 * embedDim + 0] = -R;
2594         coordsIn[4 * embedDim + 1] = R;
2595         coordsIn[4 * embedDim + 2] = R;
2596         coordsIn[5 * embedDim + 0] = R;
2597         coordsIn[5 * embedDim + 1] = R;
2598         coordsIn[5 * embedDim + 2] = R;
2599         coordsIn[6 * embedDim + 0] = -R;
2600         coordsIn[6 * embedDim + 1] = -R;
2601         coordsIn[6 * embedDim + 2] = R;
2602         coordsIn[7 * embedDim + 0] = R;
2603         coordsIn[7 * embedDim + 1] = -R;
2604         coordsIn[7 * embedDim + 2] = R;
2605       }
2606     }
2607     break;
2608   case 3:
2609     if (simplex) {
2610       const PetscReal edgeLen         = 1.0 / PETSC_PHI;
2611       PetscReal       vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
2612       PetscReal       vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
2613       PetscReal       vertexC[4]      = {0.5, 0.5 * PETSC_PHI, 0.5 / PETSC_PHI, 0.0};
2614       const PetscInt  degree          = 12;
2615       PetscInt        s[4]            = {1, 1, 1};
2616       PetscInt        evenPerm[12][4] = {
2617         {0, 1, 2, 3},
2618         {0, 2, 3, 1},
2619         {0, 3, 1, 2},
2620         {1, 0, 3, 2},
2621         {1, 2, 0, 3},
2622         {1, 3, 2, 0},
2623         {2, 0, 1, 3},
2624         {2, 1, 3, 0},
2625         {2, 3, 0, 1},
2626         {3, 0, 2, 1},
2627         {3, 1, 0, 2},
2628         {3, 2, 1, 0}
2629       };
2630       PetscInt  cone[4];
2631       PetscInt *graph, p, i, j, k, l;
2632 
2633       vertexA[0] *= R;
2634       vertexA[1] *= R;
2635       vertexA[2] *= R;
2636       vertexA[3] *= R;
2637       vertexB[0] *= R;
2638       vertexB[1] *= R;
2639       vertexB[2] *= R;
2640       vertexB[3] *= R;
2641       vertexC[0] *= R;
2642       vertexC[1] *= R;
2643       vertexC[2] *= R;
2644       vertexC[3] *= R;
2645       numCells    = rank == 0 ? 600 : 0;
2646       numVerts    = rank == 0 ? 120 : 0;
2647       firstVertex = numCells;
2648       /* Use the 600-cell, which for a unit sphere has coordinates which are
2649 
2650            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
2651                (\pm 1,    0,       0,      0)  all cyclic permutations   8
2652            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
2653 
2654          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2655          length is then given by 1/\phi = 0.61803.
2656 
2657          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
2658          http://mathworld.wolfram.com/600-Cell.html
2659       */
2660       /* Construct vertices */
2661       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2662       i = 0;
2663       if (rank == 0) {
2664         for (s[0] = -1; s[0] < 2; s[0] += 2) {
2665           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2666             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2667               for (s[3] = -1; s[3] < 2; s[3] += 2) {
2668                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[d] * vertexA[d];
2669                 ++i;
2670               }
2671             }
2672           }
2673         }
2674         for (p = 0; p < embedDim; ++p) {
2675           s[1] = s[2] = s[3] = 1;
2676           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2677             for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertexB[(d + p) % embedDim];
2678             ++i;
2679           }
2680         }
2681         for (p = 0; p < 12; ++p) {
2682           s[3] = 1;
2683           for (s[0] = -1; s[0] < 2; s[0] += 2) {
2684             for (s[1] = -1; s[1] < 2; s[1] += 2) {
2685               for (s[2] = -1; s[2] < 2; s[2] += 2) {
2686                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[evenPerm[p][d]] * vertexC[evenPerm[p][d]];
2687                 ++i;
2688               }
2689             }
2690           }
2691         }
2692       }
2693       PetscCheck(i == numVerts, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertices %" PetscInt_FMT " != %" PetscInt_FMT, i, numVerts);
2694       /* Construct graph */
2695       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2696       for (i = 0; i < numVerts; ++i) {
2697         for (j = 0, k = 0; j < numVerts; ++j) {
2698           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2699             graph[i * numVerts + j] = 1;
2700             ++k;
2701           }
2702         }
2703         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2704       }
2705       /* Build Topology */
2706       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2707       for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2708       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2709       /* Cells */
2710       if (rank == 0) {
2711         for (i = 0, c = 0; i < numVerts; ++i) {
2712           for (j = 0; j < i; ++j) {
2713             for (k = 0; k < j; ++k) {
2714               for (l = 0; l < k; ++l) {
2715                 if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i] && graph[l * numVerts + i] && graph[l * numVerts + j] && graph[l * numVerts + k]) {
2716                   cone[0] = firstVertex + i;
2717                   cone[1] = firstVertex + j;
2718                   cone[2] = firstVertex + k;
2719                   cone[3] = firstVertex + l;
2720                   /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
2721                   {
2722                     const PetscInt epsilon[4][4][4][4] = {
2723                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, -1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 0, 0}, {0, 1, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 1, 0}, {0, -1, 0, 0}, {0, 0, 0, 0}}},
2724 
2725                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}}, {{0, 0, -1, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}}},
2726 
2727                       {{{0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, 0, 0}, {0, -1, 0, 0}}, {{0, 0, 0, -1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 1, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}},
2728 
2729                       {{{0, 0, 0, 0}, {0, 0, -1, 0}, {0, 1, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 1, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}}, {{0, -1, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}} }
2730                     };
2731                     PetscReal normal[4];
2732                     PetscInt  e, f, g;
2733 
2734                     for (d = 0; d < embedDim; ++d) {
2735                       normal[d] = 0.0;
2736                       for (e = 0; e < embedDim; ++e) {
2737                         for (f = 0; f < embedDim; ++f) {
2738                           for (g = 0; g < embedDim; ++g) {
2739                             normal[d] += epsilon[d][e][f][g] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]) * (coordsIn[l * embedDim + f] - coordsIn[i * embedDim + f]);
2740                           }
2741                         }
2742                       }
2743                     }
2744                     if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2745                       PetscInt tmp = cone[1];
2746                       cone[1]      = cone[2];
2747                       cone[2]      = tmp;
2748                     }
2749                   }
2750                   PetscCall(DMPlexSetCone(dm, c++, cone));
2751                 }
2752               }
2753             }
2754           }
2755         }
2756       }
2757       PetscCall(DMPlexSymmetrize(dm));
2758       PetscCall(DMPlexStratify(dm));
2759       PetscCall(PetscFree(graph));
2760     }
2761     break;
2762   default:
2763     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unsupported dimension for sphere: %" PetscInt_FMT, dim);
2764   }
2765   /* Create coordinates */
2766   PetscCall(DMGetCoordinateSection(dm, &coordSection));
2767   PetscCall(PetscSectionSetNumFields(coordSection, 1));
2768   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, embedDim));
2769   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVerts));
2770   for (v = firstVertex; v < firstVertex + numVerts; ++v) {
2771     PetscCall(PetscSectionSetDof(coordSection, v, embedDim));
2772     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, embedDim));
2773   }
2774   PetscCall(PetscSectionSetUp(coordSection));
2775   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2776   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2777   PetscCall(VecSetBlockSize(coordinates, embedDim));
2778   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2779   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2780   PetscCall(VecSetType(coordinates, VECSTANDARD));
2781   PetscCall(VecGetArray(coordinates, &coords));
2782   for (v = 0; v < numVerts; ++v)
2783     for (d = 0; d < embedDim; ++d) coords[v * embedDim + d] = coordsIn[v * embedDim + d];
2784   PetscCall(VecRestoreArray(coordinates, &coords));
2785   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2786   PetscCall(VecDestroy(&coordinates));
2787   PetscCall(PetscFree(coordsIn));
2788   {
2789     DM          cdm;
2790     PetscDS     cds;
2791     PetscScalar c = R;
2792 
2793     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, snapToSphere));
2794     PetscCall(DMGetCoordinateDM(dm, &cdm));
2795     PetscCall(DMGetDS(cdm, &cds));
2796     PetscCall(PetscDSSetConstants(cds, 1, &c));
2797   }
2798   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
2799   /* Wait for coordinate creation before doing in-place modification */
2800   if (simplex) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2801   PetscFunctionReturn(PETSC_SUCCESS);
2802 }
2803 
2804 typedef void (*TPSEvaluateFunc)(const PetscReal[], PetscReal *, PetscReal[], PetscReal (*)[3]);
2805 
2806 /*
2807  The Schwarz P implicit surface is
2808 
2809      f(x) = cos(x0) + cos(x1) + cos(x2) = 0
2810 */
2811 static void TPSEvaluate_SchwarzP(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2812 {
2813   PetscReal c[3] = {PetscCosReal(y[0] * PETSC_PI), PetscCosReal(y[1] * PETSC_PI), PetscCosReal(y[2] * PETSC_PI)};
2814   PetscReal g[3] = {-PetscSinReal(y[0] * PETSC_PI), -PetscSinReal(y[1] * PETSC_PI), -PetscSinReal(y[2] * PETSC_PI)};
2815   f[0]           = c[0] + c[1] + c[2];
2816   for (PetscInt i = 0; i < 3; i++) {
2817     grad[i] = PETSC_PI * g[i];
2818     for (PetscInt j = 0; j < 3; j++) hess[i][j] = (i == j) ? -PetscSqr(PETSC_PI) * c[i] : 0.;
2819   }
2820 }
2821 
2822 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2823 static PetscErrorCode TPSExtrudeNormalFunc_SchwarzP(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2824 {
2825   for (PetscInt i = 0; i < 3; i++) u[i] = -PETSC_PI * PetscSinReal(x[i] * PETSC_PI);
2826   return PETSC_SUCCESS;
2827 }
2828 
2829 /*
2830  The Gyroid implicit surface is
2831 
2832  f(x,y,z) = sin(pi * x) * cos (pi * (y + 1/2))  + sin(pi * (y + 1/2)) * cos(pi * (z + 1/4)) + sin(pi * (z + 1/4)) * cos(pi * x)
2833 
2834 */
2835 static void TPSEvaluate_Gyroid(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
2836 {
2837   PetscReal s[3] = {PetscSinReal(PETSC_PI * y[0]), PetscSinReal(PETSC_PI * (y[1] + .5)), PetscSinReal(PETSC_PI * (y[2] + .25))};
2838   PetscReal c[3] = {PetscCosReal(PETSC_PI * y[0]), PetscCosReal(PETSC_PI * (y[1] + .5)), PetscCosReal(PETSC_PI * (y[2] + .25))};
2839   f[0]           = s[0] * c[1] + s[1] * c[2] + s[2] * c[0];
2840   grad[0]        = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2841   grad[1]        = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2842   grad[2]        = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2843   hess[0][0]     = -PetscSqr(PETSC_PI) * (s[0] * c[1] + s[2] * c[0]);
2844   hess[0][1]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2845   hess[0][2]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2846   hess[1][0]     = -PetscSqr(PETSC_PI) * (s[1] * c[2] + s[0] * c[1]);
2847   hess[1][1]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2848   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
2849   hess[2][0]     = -PetscSqr(PETSC_PI) * (s[2] * c[0] + s[1] * c[2]);
2850   hess[2][1]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
2851   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
2852 }
2853 
2854 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
2855 static PetscErrorCode TPSExtrudeNormalFunc_Gyroid(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
2856 {
2857   PetscReal s[3] = {PetscSinReal(PETSC_PI * x[0]), PetscSinReal(PETSC_PI * (x[1] + .5)), PetscSinReal(PETSC_PI * (x[2] + .25))};
2858   PetscReal c[3] = {PetscCosReal(PETSC_PI * x[0]), PetscCosReal(PETSC_PI * (x[1] + .5)), PetscCosReal(PETSC_PI * (x[2] + .25))};
2859   u[0]           = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
2860   u[1]           = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
2861   u[2]           = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
2862   return PETSC_SUCCESS;
2863 }
2864 
2865 /*
2866    We wish to solve
2867 
2868          min_y || y - x ||^2  subject to f(y) = 0
2869 
2870    Let g(y) = grad(f).  The minimization problem is equivalent to asking to satisfy
2871    f(y) = 0 and (y-x) is parallel to g(y).  We do this by using Householder QR to obtain a basis for the
2872    tangent space and ask for both components in the tangent space to be zero.
2873 
2874    Take g to be a column vector and compute the "full QR" factorization Q R = g,
2875    where Q = I - 2 n n^T is a symmetric orthogonal matrix.
2876    The first column of Q is parallel to g so the remaining two columns span the null space.
2877    Let Qn = Q[:,1:] be those remaining columns.  Then Qn Qn^T is an orthogonal projector into the tangent space.
2878    Since Q is symmetric, this is equivalent to multiplying by Q and taking the last two entries.
2879    In total, we have a system of 3 equations in 3 unknowns:
2880 
2881      f(y) = 0                       1 equation
2882      Qn^T (y - x) = 0               2 equations
2883 
2884    Here, we compute the residual and Jacobian of this system.
2885 */
2886 static void TPSNearestPointResJac(TPSEvaluateFunc feval, const PetscScalar x[], const PetscScalar y[], PetscScalar res[], PetscScalar J[])
2887 {
2888   PetscReal yreal[3] = {PetscRealPart(y[0]), PetscRealPart(y[1]), PetscRealPart(y[2])};
2889   PetscReal d[3]     = {PetscRealPart(y[0] - x[0]), PetscRealPart(y[1] - x[1]), PetscRealPart(y[2] - x[2])};
2890   PetscReal f, grad[3], n[3], norm, norm_y[3], nd, nd_y[3], sign;
2891   PetscReal n_y[3][3] = {
2892     {0, 0, 0},
2893     {0, 0, 0},
2894     {0, 0, 0}
2895   };
2896 
2897   feval(yreal, &f, grad, n_y);
2898 
2899   for (PetscInt i = 0; i < 3; i++) n[i] = grad[i];
2900   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2901   for (PetscInt i = 0; i < 3; i++) norm_y[i] = 1. / norm * n[i] * n_y[i][i];
2902 
2903   // Define the Householder reflector
2904   sign = n[0] >= 0 ? 1. : -1.;
2905   n[0] += norm * sign;
2906   for (PetscInt i = 0; i < 3; i++) n_y[0][i] += norm_y[i] * sign;
2907 
2908   norm      = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
2909   norm_y[0] = 1. / norm * (n[0] * n_y[0][0]);
2910   norm_y[1] = 1. / norm * (n[0] * n_y[0][1] + n[1] * n_y[1][1]);
2911   norm_y[2] = 1. / norm * (n[0] * n_y[0][2] + n[2] * n_y[2][2]);
2912 
2913   for (PetscInt i = 0; i < 3; i++) {
2914     n[i] /= norm;
2915     for (PetscInt j = 0; j < 3; j++) {
2916       // note that n[i] is n_old[i]/norm when executing the code below
2917       n_y[i][j] = n_y[i][j] / norm - n[i] / norm * norm_y[j];
2918     }
2919   }
2920 
2921   nd = n[0] * d[0] + n[1] * d[1] + n[2] * d[2];
2922   for (PetscInt i = 0; i < 3; i++) nd_y[i] = n[i] + n_y[0][i] * d[0] + n_y[1][i] * d[1] + n_y[2][i] * d[2];
2923 
2924   res[0] = f;
2925   res[1] = d[1] - 2 * n[1] * nd;
2926   res[2] = d[2] - 2 * n[2] * nd;
2927   // J[j][i] is J_{ij} (column major)
2928   for (PetscInt j = 0; j < 3; j++) {
2929     J[0 + j * 3] = grad[j];
2930     J[1 + j * 3] = (j == 1) * 1. - 2 * (n_y[1][j] * nd + n[1] * nd_y[j]);
2931     J[2 + j * 3] = (j == 2) * 1. - 2 * (n_y[2][j] * nd + n[2] * nd_y[j]);
2932   }
2933 }
2934 
2935 /*
2936    Project x to the nearest point on the implicit surface using Newton's method.
2937 */
2938 static PetscErrorCode TPSNearestPoint(TPSEvaluateFunc feval, PetscScalar x[])
2939 {
2940   PetscScalar y[3] = {x[0], x[1], x[2]}; // Initial guess
2941 
2942   PetscFunctionBegin;
2943   for (PetscInt iter = 0; iter < 10; iter++) {
2944     PetscScalar res[3], J[9];
2945     PetscReal   resnorm;
2946     TPSNearestPointResJac(feval, x, y, res, J);
2947     resnorm = PetscSqrtReal(PetscSqr(PetscRealPart(res[0])) + PetscSqr(PetscRealPart(res[1])) + PetscSqr(PetscRealPart(res[2])));
2948     if (0) { // Turn on this monitor if you need to confirm quadratic convergence
2949       PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%" PetscInt_FMT "] res [%g %g %g]\n", iter, (double)PetscRealPart(res[0]), (double)PetscRealPart(res[1]), (double)PetscRealPart(res[2])));
2950     }
2951     if (resnorm < PETSC_SMALL) break;
2952 
2953     // Take the Newton step
2954     PetscCall(PetscKernel_A_gets_inverse_A_3(J, 0., PETSC_FALSE, NULL));
2955     PetscKernel_v_gets_v_minus_A_times_w_3(y, J, res);
2956   }
2957   for (PetscInt i = 0; i < 3; i++) x[i] = y[i];
2958   PetscFunctionReturn(PETSC_SUCCESS);
2959 }
2960 
2961 const char *const DMPlexTPSTypes[] = {"SCHWARZ_P", "GYROID", "DMPlexTPSType", "DMPLEX_TPS_", NULL};
2962 
2963 static PetscErrorCode DMPlexCreateTPSMesh_Internal(DM dm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness)
2964 {
2965   PetscMPIInt rank;
2966   PetscInt    topoDim = 2, spaceDim = 3, numFaces = 0, numVertices = 0, numEdges = 0;
2967   PetscInt(*edges)[2] = NULL, *edgeSets = NULL;
2968   PetscInt            *cells_flat = NULL;
2969   PetscReal           *vtxCoords  = NULL;
2970   TPSEvaluateFunc      evalFunc   = NULL;
2971   PetscSimplePointFunc normalFunc = NULL;
2972   DMLabel              label;
2973 
2974   PetscFunctionBegin;
2975   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
2976   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2977   PetscCheck((layers != 0) ^ (thickness == 0.), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Layers %" PetscInt_FMT " must be nonzero iff thickness %g is nonzero", layers, (double)thickness);
2978   switch (tpstype) {
2979   case DMPLEX_TPS_SCHWARZ_P:
2980     PetscCheck(!periodic || (periodic[0] == DM_BOUNDARY_NONE && periodic[1] == DM_BOUNDARY_NONE && periodic[2] == DM_BOUNDARY_NONE), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Schwarz P does not support periodic meshes");
2981     if (rank == 0) {
2982       PetscInt(*cells)[6][4][4] = NULL; // [junction, junction-face, cell, conn]
2983       PetscInt  Njunctions = 0, Ncuts = 0, Npipes[3], vcount;
2984       PetscReal L = 1;
2985 
2986       Npipes[0]   = (extent[0] + 1) * extent[1] * extent[2];
2987       Npipes[1]   = extent[0] * (extent[1] + 1) * extent[2];
2988       Npipes[2]   = extent[0] * extent[1] * (extent[2] + 1);
2989       Njunctions  = extent[0] * extent[1] * extent[2];
2990       Ncuts       = 2 * (extent[0] * extent[1] + extent[1] * extent[2] + extent[2] * extent[0]);
2991       numVertices = 4 * (Npipes[0] + Npipes[1] + Npipes[2]) + 8 * Njunctions;
2992       PetscCall(PetscMalloc1(3 * numVertices, &vtxCoords));
2993       PetscCall(PetscMalloc1(Njunctions, &cells));
2994       PetscCall(PetscMalloc1(Ncuts * 4, &edges));
2995       PetscCall(PetscMalloc1(Ncuts * 4, &edgeSets));
2996       // x-normal pipes
2997       vcount = 0;
2998       for (PetscInt i = 0; i < extent[0] + 1; i++) {
2999         for (PetscInt j = 0; j < extent[1]; j++) {
3000           for (PetscInt k = 0; k < extent[2]; k++) {
3001             for (PetscInt l = 0; l < 4; l++) {
3002               vtxCoords[vcount++] = (2 * i - 1) * L;
3003               vtxCoords[vcount++] = 2 * j * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3004               vtxCoords[vcount++] = 2 * k * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3005             }
3006           }
3007         }
3008       }
3009       // y-normal pipes
3010       for (PetscInt i = 0; i < extent[0]; i++) {
3011         for (PetscInt j = 0; j < extent[1] + 1; j++) {
3012           for (PetscInt k = 0; k < extent[2]; k++) {
3013             for (PetscInt l = 0; l < 4; l++) {
3014               vtxCoords[vcount++] = 2 * i * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3015               vtxCoords[vcount++] = (2 * j - 1) * L;
3016               vtxCoords[vcount++] = 2 * k * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3017             }
3018           }
3019         }
3020       }
3021       // z-normal pipes
3022       for (PetscInt i = 0; i < extent[0]; i++) {
3023         for (PetscInt j = 0; j < extent[1]; j++) {
3024           for (PetscInt k = 0; k < extent[2] + 1; k++) {
3025             for (PetscInt l = 0; l < 4; l++) {
3026               vtxCoords[vcount++] = 2 * i * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3027               vtxCoords[vcount++] = 2 * j * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3028               vtxCoords[vcount++] = (2 * k - 1) * L;
3029             }
3030           }
3031         }
3032       }
3033       // junctions
3034       for (PetscInt i = 0; i < extent[0]; i++) {
3035         for (PetscInt j = 0; j < extent[1]; j++) {
3036           for (PetscInt k = 0; k < extent[2]; k++) {
3037             const PetscInt J = (i * extent[1] + j) * extent[2] + k, Jvoff = (Npipes[0] + Npipes[1] + Npipes[2]) * 4 + J * 8;
3038             PetscCheck(vcount / 3 == Jvoff, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected vertex count");
3039             for (PetscInt ii = 0; ii < 2; ii++) {
3040               for (PetscInt jj = 0; jj < 2; jj++) {
3041                 for (PetscInt kk = 0; kk < 2; kk++) {
3042                   double Ls           = (1 - sqrt(2) / 4) * L;
3043                   vtxCoords[vcount++] = 2 * i * L + (2 * ii - 1) * Ls;
3044                   vtxCoords[vcount++] = 2 * j * L + (2 * jj - 1) * Ls;
3045                   vtxCoords[vcount++] = 2 * k * L + (2 * kk - 1) * Ls;
3046                 }
3047               }
3048             }
3049             const PetscInt jfaces[3][2][4] = {
3050               {{3, 1, 0, 2}, {7, 5, 4, 6}}, // x-aligned
3051               {{5, 4, 0, 1}, {7, 6, 2, 3}}, // y-aligned
3052               {{6, 2, 0, 4}, {7, 3, 1, 5}}  // z-aligned
3053             };
3054             const PetscInt pipe_lo[3] = {// vertex numbers of pipes
3055                                          ((i * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + Npipes[0] + Npipes[1]) * 4};
3056             const PetscInt pipe_hi[3] = {// vertex numbers of pipes
3057                                          (((i + 1) * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j + 1) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + 1 + Npipes[0] + Npipes[1]) * 4};
3058             for (PetscInt dir = 0; dir < 3; dir++) { // x,y,z
3059               const PetscInt ijk[3] = {i, j, k};
3060               for (PetscInt l = 0; l < 4; l++) { // rotations
3061                 cells[J][dir * 2 + 0][l][0] = pipe_lo[dir] + l;
3062                 cells[J][dir * 2 + 0][l][1] = Jvoff + jfaces[dir][0][l];
3063                 cells[J][dir * 2 + 0][l][2] = Jvoff + jfaces[dir][0][(l - 1 + 4) % 4];
3064                 cells[J][dir * 2 + 0][l][3] = pipe_lo[dir] + (l - 1 + 4) % 4;
3065                 cells[J][dir * 2 + 1][l][0] = Jvoff + jfaces[dir][1][l];
3066                 cells[J][dir * 2 + 1][l][1] = pipe_hi[dir] + l;
3067                 cells[J][dir * 2 + 1][l][2] = pipe_hi[dir] + (l - 1 + 4) % 4;
3068                 cells[J][dir * 2 + 1][l][3] = Jvoff + jfaces[dir][1][(l - 1 + 4) % 4];
3069                 if (ijk[dir] == 0) {
3070                   edges[numEdges][0] = pipe_lo[dir] + l;
3071                   edges[numEdges][1] = pipe_lo[dir] + (l + 1) % 4;
3072                   edgeSets[numEdges] = dir * 2 + 1;
3073                   numEdges++;
3074                 }
3075                 if (ijk[dir] + 1 == extent[dir]) {
3076                   edges[numEdges][0] = pipe_hi[dir] + l;
3077                   edges[numEdges][1] = pipe_hi[dir] + (l + 1) % 4;
3078                   edgeSets[numEdges] = dir * 2 + 2;
3079                   numEdges++;
3080                 }
3081               }
3082             }
3083           }
3084         }
3085       }
3086       PetscCheck(numEdges == Ncuts * 4, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Edge count %" PetscInt_FMT " incompatible with number of cuts %" PetscInt_FMT, numEdges, Ncuts);
3087       numFaces   = 24 * Njunctions;
3088       cells_flat = cells[0][0][0];
3089     }
3090     evalFunc   = TPSEvaluate_SchwarzP;
3091     normalFunc = TPSExtrudeNormalFunc_SchwarzP;
3092     break;
3093   case DMPLEX_TPS_GYROID:
3094     if (rank == 0) {
3095       // This is a coarse mesh approximation of the gyroid shifted to being the zero of the level set
3096       //
3097       //     sin(pi*x)*cos(pi*(y+1/2)) + sin(pi*(y+1/2))*cos(pi*(z+1/4)) + sin(pi*(z+1/4))*cos(x)
3098       //
3099       // on the cell [0,2]^3.
3100       //
3101       // Think about dividing that cell into four columns, and focus on the column [0,1]x[0,1]x[0,2].
3102       // If you looked at the gyroid in that column at different slices of z you would see that it kind of spins
3103       // like a boomerang:
3104       //
3105       //     z = 0          z = 1/4        z = 1/2        z = 3/4     //
3106       //     -----          -------        -------        -------     //
3107       //                                                              //
3108       //     +       +      +       +      +       +      +   \   +   //
3109       //      \                                   /            \      //
3110       //       \            `-_   _-'            /              }     //
3111       //        *-_            `-'            _-'              /      //
3112       //     +     `-+      +       +      +-'     +      +   /   +   //
3113       //                                                              //
3114       //                                                              //
3115       //     z = 1          z = 5/4        z = 3/2        z = 7/4     //
3116       //     -----          -------        -------        -------     //
3117       //                                                              //
3118       //     +-_     +      +       +      +     _-+      +   /   +   //
3119       //        `-_            _-_            _-`            /        //
3120       //           \        _-'   `-_        /              {         //
3121       //            \                       /                \        //
3122       //     +       +      +       +      +       +      +   \   +   //
3123       //
3124       //
3125       // This course mesh approximates each of these slices by two line segments,
3126       // and then connects the segments in consecutive layers with quadrilateral faces.
3127       // All of the end points of the segments are multiples of 1/4 except for the
3128       // point * in the picture for z = 0 above and the similar points in other layers.
3129       // That point is at (gamma, gamma, 0), where gamma is calculated below.
3130       //
3131       // The column  [1,2]x[1,2]x[0,2] looks the same as this column;
3132       // The columns [1,2]x[0,1]x[0,2] and [0,1]x[1,2]x[0,2] are mirror images.
3133       //
3134       // As for how this method turned into the names given to the vertices:
3135       // that was not systematic, it was just the way it worked out in my handwritten notes.
3136 
3137       PetscInt facesPerBlock = 64;
3138       PetscInt vertsPerBlock = 56;
3139       PetscInt extentPlus[3];
3140       PetscInt numBlocks, numBlocksPlus;
3141       const PetscInt A = 0, B = 1, C = 2, D = 3, E = 4, F = 5, G = 6, H = 7, II = 8, J = 9, K = 10, L = 11, M = 12, N = 13, O = 14, P = 15, Q = 16, R = 17, S = 18, T = 19, U = 20, V = 21, W = 22, X = 23, Y = 24, Z = 25, Ap = 26, Bp = 27, Cp = 28, Dp = 29, Ep = 30, Fp = 31, Gp = 32, Hp = 33, Ip = 34, Jp = 35, Kp = 36, Lp = 37, Mp = 38, Np = 39, Op = 40, Pp = 41, Qp = 42, Rp = 43, Sp = 44, Tp = 45, Up = 46, Vp = 47, Wp = 48, Xp = 49, Yp = 50, Zp = 51, Aq = 52, Bq = 53, Cq = 54, Dq = 55;
3142       const PetscInt pattern[64][4] = {
3143   /* face to vertex within the coarse discretization of a single gyroid block */
3144   /* layer 0 */
3145         {A,           C,           K,           G          },
3146         {C,           B,           II,          K          },
3147         {D,           A,           H,           L          },
3148         {B + 56 * 1,  D,           L,           J          },
3149         {E,           B + 56 * 1,  J,           N          },
3150         {A + 56 * 2,  E,           N,           H + 56 * 2 },
3151         {F,           A + 56 * 2,  G + 56 * 2,  M          },
3152         {B,           F,           M,           II         },
3153  /* layer 1 */
3154         {G,           K,           Q,           O          },
3155         {K,           II,          P,           Q          },
3156         {L,           H,           O + 56 * 1,  R          },
3157         {J,           L,           R,           P          },
3158         {N,           J,           P,           S          },
3159         {H + 56 * 2,  N,           S,           O + 56 * 3 },
3160         {M,           G + 56 * 2,  O + 56 * 2,  T          },
3161         {II,          M,           T,           P          },
3162  /* layer 2 */
3163         {O,           Q,           Y,           U          },
3164         {Q,           P,           W,           Y          },
3165         {R,           O + 56 * 1,  U + 56 * 1,  Ap         },
3166         {P,           R,           Ap,          W          },
3167         {S,           P,           X,           Bp         },
3168         {O + 56 * 3,  S,           Bp,          V + 56 * 1 },
3169         {T,           O + 56 * 2,  V,           Z          },
3170         {P,           T,           Z,           X          },
3171  /* layer 3 */
3172         {U,           Y,           Ep,          Dp         },
3173         {Y,           W,           Cp,          Ep         },
3174         {Ap,          U + 56 * 1,  Dp + 56 * 1, Gp         },
3175         {W,           Ap,          Gp,          Cp         },
3176         {Bp,          X,           Cp + 56 * 2, Fp         },
3177         {V + 56 * 1,  Bp,          Fp,          Dp + 56 * 1},
3178         {Z,           V,           Dp,          Hp         },
3179         {X,           Z,           Hp,          Cp + 56 * 2},
3180  /* layer 4 */
3181         {Dp,          Ep,          Mp,          Kp         },
3182         {Ep,          Cp,          Ip,          Mp         },
3183         {Gp,          Dp + 56 * 1, Lp,          Np         },
3184         {Cp,          Gp,          Np,          Jp         },
3185         {Fp,          Cp + 56 * 2, Jp + 56 * 2, Pp         },
3186         {Dp + 56 * 1, Fp,          Pp,          Lp         },
3187         {Hp,          Dp,          Kp,          Op         },
3188         {Cp + 56 * 2, Hp,          Op,          Ip + 56 * 2},
3189  /* layer 5 */
3190         {Kp,          Mp,          Sp,          Rp         },
3191         {Mp,          Ip,          Qp,          Sp         },
3192         {Np,          Lp,          Rp,          Tp         },
3193         {Jp,          Np,          Tp,          Qp + 56 * 1},
3194         {Pp,          Jp + 56 * 2, Qp + 56 * 3, Up         },
3195         {Lp,          Pp,          Up,          Rp         },
3196         {Op,          Kp,          Rp,          Vp         },
3197         {Ip + 56 * 2, Op,          Vp,          Qp + 56 * 2},
3198  /* layer 6 */
3199         {Rp,          Sp,          Aq,          Yp         },
3200         {Sp,          Qp,          Wp,          Aq         },
3201         {Tp,          Rp,          Yp,          Cq         },
3202         {Qp + 56 * 1, Tp,          Cq,          Wp + 56 * 1},
3203         {Up,          Qp + 56 * 3, Xp + 56 * 1, Dq         },
3204         {Rp,          Up,          Dq,          Zp         },
3205         {Vp,          Rp,          Zp,          Bq         },
3206         {Qp + 56 * 2, Vp,          Bq,          Xp         },
3207  /* layer 7 (the top is the periodic image of the bottom of layer 0) */
3208         {Yp,          Aq,          C + 56 * 4,  A + 56 * 4 },
3209         {Aq,          Wp,          B + 56 * 4,  C + 56 * 4 },
3210         {Cq,          Yp,          A + 56 * 4,  D + 56 * 4 },
3211         {Wp + 56 * 1, Cq,          D + 56 * 4,  B + 56 * 5 },
3212         {Dq,          Xp + 56 * 1, B + 56 * 5,  E + 56 * 4 },
3213         {Zp,          Dq,          E + 56 * 4,  A + 56 * 6 },
3214         {Bq,          Zp,          A + 56 * 6,  F + 56 * 4 },
3215         {Xp,          Bq,          F + 56 * 4,  B + 56 * 4 }
3216       };
3217       const PetscReal gamma                = PetscAcosReal((PetscSqrtReal(3.) - 1.) / PetscSqrtReal(2.)) / PETSC_PI;
3218       const PetscReal patternCoords[56][3] = {
3219         {1.,        0.,        0.  }, /* A  */
3220         {0.,        1.,        0.  }, /* B  */
3221         {gamma,     gamma,     0.  }, /* C  */
3222         {1 + gamma, 1 - gamma, 0.  }, /* D  */
3223         {2 - gamma, 2 - gamma, 0.  }, /* E  */
3224         {1 - gamma, 1 + gamma, 0.  }, /* F  */
3225 
3226         {.5,        0,         .25 }, /* G  */
3227         {1.5,       0.,        .25 }, /* H  */
3228         {.5,        1.,        .25 }, /* II */
3229         {1.5,       1.,        .25 }, /* J  */
3230         {.25,       .5,        .25 }, /* K  */
3231         {1.25,      .5,        .25 }, /* L  */
3232         {.75,       1.5,       .25 }, /* M  */
3233         {1.75,      1.5,       .25 }, /* N  */
3234 
3235         {0.,        0.,        .5  }, /* O  */
3236         {1.,        1.,        .5  }, /* P  */
3237         {gamma,     1 - gamma, .5  }, /* Q  */
3238         {1 + gamma, gamma,     .5  }, /* R  */
3239         {2 - gamma, 1 + gamma, .5  }, /* S  */
3240         {1 - gamma, 2 - gamma, .5  }, /* T  */
3241 
3242         {0.,        .5,        .75 }, /* U  */
3243         {0.,        1.5,       .75 }, /* V  */
3244         {1.,        .5,        .75 }, /* W  */
3245         {1.,        1.5,       .75 }, /* X  */
3246         {.5,        .75,       .75 }, /* Y  */
3247         {.5,        1.75,      .75 }, /* Z  */
3248         {1.5,       .25,       .75 }, /* Ap */
3249         {1.5,       1.25,      .75 }, /* Bp */
3250 
3251         {1.,        0.,        1.  }, /* Cp */
3252         {0.,        1.,        1.  }, /* Dp */
3253         {1 - gamma, 1 - gamma, 1.  }, /* Ep */
3254         {1 + gamma, 1 + gamma, 1.  }, /* Fp */
3255         {2 - gamma, gamma,     1.  }, /* Gp */
3256         {gamma,     2 - gamma, 1.  }, /* Hp */
3257 
3258         {.5,        0.,        1.25}, /* Ip */
3259         {1.5,       0.,        1.25}, /* Jp */
3260         {.5,        1.,        1.25}, /* Kp */
3261         {1.5,       1.,        1.25}, /* Lp */
3262         {.75,       .5,        1.25}, /* Mp */
3263         {1.75,      .5,        1.25}, /* Np */
3264         {.25,       1.5,       1.25}, /* Op */
3265         {1.25,      1.5,       1.25}, /* Pp */
3266 
3267         {0.,        0.,        1.5 }, /* Qp */
3268         {1.,        1.,        1.5 }, /* Rp */
3269         {1 - gamma, gamma,     1.5 }, /* Sp */
3270         {2 - gamma, 1 - gamma, 1.5 }, /* Tp */
3271         {1 + gamma, 2 - gamma, 1.5 }, /* Up */
3272         {gamma,     1 + gamma, 1.5 }, /* Vp */
3273 
3274         {0.,        .5,        1.75}, /* Wp */
3275         {0.,        1.5,       1.75}, /* Xp */
3276         {1.,        .5,        1.75}, /* Yp */
3277         {1.,        1.5,       1.75}, /* Zp */
3278         {.5,        .25,       1.75}, /* Aq */
3279         {.5,        1.25,      1.75}, /* Bq */
3280         {1.5,       .75,       1.75}, /* Cq */
3281         {1.5,       1.75,      1.75}, /* Dq */
3282       };
3283       PetscInt(*cells)[64][4] = NULL;
3284       PetscBool *seen;
3285       PetscInt  *vertToTrueVert;
3286       PetscInt   count;
3287 
3288       for (PetscInt i = 0; i < 3; i++) extentPlus[i] = extent[i] + 1;
3289       numBlocks = 1;
3290       for (PetscInt i = 0; i < 3; i++) numBlocks *= extent[i];
3291       numBlocksPlus = 1;
3292       for (PetscInt i = 0; i < 3; i++) numBlocksPlus *= extentPlus[i];
3293       numFaces = numBlocks * facesPerBlock;
3294       PetscCall(PetscMalloc1(numBlocks, &cells));
3295       PetscCall(PetscCalloc1(numBlocksPlus * vertsPerBlock, &seen));
3296       for (PetscInt k = 0; k < extent[2]; k++) {
3297         for (PetscInt j = 0; j < extent[1]; j++) {
3298           for (PetscInt i = 0; i < extent[0]; i++) {
3299             for (PetscInt f = 0; f < facesPerBlock; f++) {
3300               for (PetscInt v = 0; v < 4; v++) {
3301                 PetscInt vertRaw     = pattern[f][v];
3302                 PetscInt blockidx    = vertRaw / 56;
3303                 PetscInt patternvert = vertRaw % 56;
3304                 PetscInt xplus       = (blockidx & 1);
3305                 PetscInt yplus       = (blockidx & 2) >> 1;
3306                 PetscInt zplus       = (blockidx & 4) >> 2;
3307                 PetscInt zcoord      = (periodic && periodic[2] == DM_BOUNDARY_PERIODIC) ? ((k + zplus) % extent[2]) : (k + zplus);
3308                 PetscInt ycoord      = (periodic && periodic[1] == DM_BOUNDARY_PERIODIC) ? ((j + yplus) % extent[1]) : (j + yplus);
3309                 PetscInt xcoord      = (periodic && periodic[0] == DM_BOUNDARY_PERIODIC) ? ((i + xplus) % extent[0]) : (i + xplus);
3310                 PetscInt vert        = ((zcoord * extentPlus[1] + ycoord) * extentPlus[0] + xcoord) * 56 + patternvert;
3311 
3312                 cells[(k * extent[1] + j) * extent[0] + i][f][v] = vert;
3313                 seen[vert]                                       = PETSC_TRUE;
3314               }
3315             }
3316           }
3317         }
3318       }
3319       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++)
3320         if (seen[i]) numVertices++;
3321       count = 0;
3322       PetscCall(PetscMalloc1(numBlocksPlus * vertsPerBlock, &vertToTrueVert));
3323       PetscCall(PetscMalloc1(numVertices * 3, &vtxCoords));
3324       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) vertToTrueVert[i] = -1;
3325       for (PetscInt k = 0; k < extentPlus[2]; k++) {
3326         for (PetscInt j = 0; j < extentPlus[1]; j++) {
3327           for (PetscInt i = 0; i < extentPlus[0]; i++) {
3328             for (PetscInt v = 0; v < vertsPerBlock; v++) {
3329               PetscInt vIdx = ((k * extentPlus[1] + j) * extentPlus[0] + i) * vertsPerBlock + v;
3330 
3331               if (seen[vIdx]) {
3332                 PetscInt thisVert;
3333 
3334                 vertToTrueVert[vIdx] = thisVert = count++;
3335 
3336                 for (PetscInt d = 0; d < 3; d++) vtxCoords[3 * thisVert + d] = patternCoords[v][d];
3337                 vtxCoords[3 * thisVert + 0] += i * 2;
3338                 vtxCoords[3 * thisVert + 1] += j * 2;
3339                 vtxCoords[3 * thisVert + 2] += k * 2;
3340               }
3341             }
3342           }
3343         }
3344       }
3345       for (PetscInt i = 0; i < numBlocks; i++) {
3346         for (PetscInt f = 0; f < facesPerBlock; f++) {
3347           for (PetscInt v = 0; v < 4; v++) cells[i][f][v] = vertToTrueVert[cells[i][f][v]];
3348         }
3349       }
3350       PetscCall(PetscFree(vertToTrueVert));
3351       PetscCall(PetscFree(seen));
3352       cells_flat = cells[0][0];
3353       numEdges   = 0;
3354       for (PetscInt i = 0; i < numFaces; i++) {
3355         for (PetscInt e = 0; e < 4; e++) {
3356           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3357           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3358 
3359           for (PetscInt d = 0; d < 3; d++) {
3360             if (!periodic || periodic[0] != DM_BOUNDARY_PERIODIC) {
3361               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) numEdges++;
3362               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) numEdges++;
3363             }
3364           }
3365         }
3366       }
3367       PetscCall(PetscMalloc1(numEdges, &edges));
3368       PetscCall(PetscMalloc1(numEdges, &edgeSets));
3369       for (PetscInt edge = 0, i = 0; i < numFaces; i++) {
3370         for (PetscInt e = 0; e < 4; e++) {
3371           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3372           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3373 
3374           for (PetscInt d = 0; d < 3; d++) {
3375             if (!periodic || periodic[d] != DM_BOUNDARY_PERIODIC) {
3376               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) {
3377                 edges[edge][0]   = ev[0];
3378                 edges[edge][1]   = ev[1];
3379                 edgeSets[edge++] = 2 * d;
3380               }
3381               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) {
3382                 edges[edge][0]   = ev[0];
3383                 edges[edge][1]   = ev[1];
3384                 edgeSets[edge++] = 2 * d + 1;
3385               }
3386             }
3387           }
3388         }
3389       }
3390     }
3391     evalFunc   = TPSEvaluate_Gyroid;
3392     normalFunc = TPSExtrudeNormalFunc_Gyroid;
3393     break;
3394   }
3395 
3396   PetscCall(DMSetDimension(dm, topoDim));
3397   if (rank == 0) PetscCall(DMPlexBuildFromCellList(dm, numFaces, numVertices, 4, cells_flat));
3398   else PetscCall(DMPlexBuildFromCellList(dm, 0, 0, 0, NULL));
3399   PetscCall(PetscFree(cells_flat));
3400   {
3401     DM idm;
3402     PetscCall(DMPlexInterpolate(dm, &idm));
3403     PetscCall(DMPlexReplace_Internal(dm, &idm));
3404   }
3405   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, vtxCoords));
3406   else PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, NULL));
3407   PetscCall(PetscFree(vtxCoords));
3408 
3409   PetscCall(DMCreateLabel(dm, "Face Sets"));
3410   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3411   for (PetscInt e = 0; e < numEdges; e++) {
3412     PetscInt        njoin;
3413     const PetscInt *join, verts[] = {numFaces + edges[e][0], numFaces + edges[e][1]};
3414     PetscCall(DMPlexGetJoin(dm, 2, verts, &njoin, &join));
3415     PetscCheck(njoin == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected unique join of vertices %" PetscInt_FMT " and %" PetscInt_FMT, edges[e][0], edges[e][1]);
3416     PetscCall(DMLabelSetValue(label, join[0], edgeSets[e]));
3417     PetscCall(DMPlexRestoreJoin(dm, 2, verts, &njoin, &join));
3418   }
3419   PetscCall(PetscFree(edges));
3420   PetscCall(PetscFree(edgeSets));
3421   if (tps_distribute) {
3422     DM               pdm = NULL;
3423     PetscPartitioner part;
3424 
3425     PetscCall(DMPlexGetPartitioner(dm, &part));
3426     PetscCall(PetscPartitionerSetFromOptions(part));
3427     PetscCall(DMPlexDistribute(dm, 0, NULL, &pdm));
3428     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
3429     // Do not auto-distribute again
3430     PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
3431   }
3432 
3433   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3434   for (PetscInt refine = 0; refine < refinements; refine++) {
3435     PetscInt     m;
3436     DM           dmf;
3437     Vec          X;
3438     PetscScalar *x;
3439     PetscCall(DMRefine(dm, MPI_COMM_NULL, &dmf));
3440     PetscCall(DMPlexReplace_Internal(dm, &dmf));
3441 
3442     PetscCall(DMGetCoordinatesLocal(dm, &X));
3443     PetscCall(VecGetLocalSize(X, &m));
3444     PetscCall(VecGetArray(X, &x));
3445     for (PetscInt i = 0; i < m; i += 3) PetscCall(TPSNearestPoint(evalFunc, &x[i]));
3446     PetscCall(VecRestoreArray(X, &x));
3447   }
3448 
3449   // Face Sets has already been propagated to new vertices during refinement; this propagates to the initial vertices.
3450   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3451   PetscCall(DMPlexLabelComplete(dm, label));
3452 
3453   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
3454 
3455   if (thickness > 0) {
3456     DM              edm, cdm, ecdm;
3457     DMPlexTransform tr;
3458     const char     *prefix;
3459     PetscOptions    options;
3460     // Code from DMPlexExtrude
3461     PetscCall(DMPlexTransformCreate(PetscObjectComm((PetscObject)dm), &tr));
3462     PetscCall(DMPlexTransformSetDM(tr, dm));
3463     PetscCall(DMPlexTransformSetType(tr, DMPLEXEXTRUDE));
3464     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
3465     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)tr, prefix));
3466     PetscCall(PetscObjectGetOptions((PetscObject)dm, &options));
3467     PetscCall(PetscObjectSetOptions((PetscObject)tr, options));
3468     PetscCall(DMPlexTransformExtrudeSetLayers(tr, layers));
3469     PetscCall(DMPlexTransformExtrudeSetThickness(tr, thickness));
3470     PetscCall(DMPlexTransformExtrudeSetTensor(tr, PETSC_FALSE));
3471     PetscCall(DMPlexTransformExtrudeSetSymmetric(tr, PETSC_TRUE));
3472     PetscCall(DMPlexTransformExtrudeSetNormalFunction(tr, normalFunc));
3473     PetscCall(DMPlexTransformSetFromOptions(tr));
3474     PetscCall(PetscObjectSetOptions((PetscObject)tr, NULL));
3475     PetscCall(DMPlexTransformSetUp(tr));
3476     PetscCall(PetscObjectViewFromOptions((PetscObject)tr, NULL, "-dm_plex_tps_transform_view"));
3477     PetscCall(DMPlexTransformApply(tr, dm, &edm));
3478     PetscCall(DMCopyDisc(dm, edm));
3479     PetscCall(DMGetCoordinateDM(dm, &cdm));
3480     PetscCall(DMGetCoordinateDM(edm, &ecdm));
3481     PetscCall(DMCopyDisc(cdm, ecdm));
3482     PetscCall(DMPlexTransformCreateDiscLabels(tr, edm));
3483     PetscCall(DMPlexTransformDestroy(&tr));
3484     if (edm) {
3485       ((DM_Plex *)edm->data)->printFEM    = ((DM_Plex *)dm->data)->printFEM;
3486       ((DM_Plex *)edm->data)->printL2     = ((DM_Plex *)dm->data)->printL2;
3487       ((DM_Plex *)edm->data)->printLocate = ((DM_Plex *)dm->data)->printLocate;
3488     }
3489     PetscCall(DMPlexReplace_Internal(dm, &edm));
3490   }
3491   PetscFunctionReturn(PETSC_SUCCESS);
3492 }
3493 
3494 /*@
3495   DMPlexCreateTPSMesh - Create a distributed, interpolated mesh of a triply-periodic surface
3496 
3497   Collective
3498 
3499   Input Parameters:
3500 + comm           - The communicator for the `DM` object
3501 . tpstype        - Type of triply-periodic surface
3502 . extent         - Array of length 3 containing number of periods in each direction
3503 . periodic       - array of length 3 with periodicity, or `NULL` for non-periodic
3504 . tps_distribute - Distribute 2D manifold mesh prior to refinement and extrusion (more scalable)
3505 . refinements    - Number of factor-of-2 refinements of 2D manifold mesh
3506 . layers         - Number of cell layers extruded in normal direction
3507 - thickness      - Thickness in normal direction
3508 
3509   Output Parameter:
3510 . dm - The `DM` object
3511 
3512   Level: beginner
3513 
3514   Notes:
3515   This meshes the surface of the Schwarz P or Gyroid surfaces.  Schwarz P is is the simplest member of the triply-periodic minimal surfaces.
3516   https://en.wikipedia.org/wiki/Schwarz_minimal_surface#Schwarz_P_(%22Primitive%22) and can be cut with "clean" boundaries.
3517   The Gyroid (https://en.wikipedia.org/wiki/Gyroid) is another triply-periodic minimal surface with applications in additive manufacturing; it is much more difficult to "cut" since there are no planes of symmetry.
3518   Our implementation creates a very coarse mesh of the surface and refines (by 4-way splitting) as many times as requested.
3519   On each refinement, all vertices are projected to their nearest point on the surface.
3520   This projection could readily be extended to related surfaces.
3521 
3522   The face (edge) sets for the Schwarz P surface are numbered 1(-x), 2(+x), 3(-y), 4(+y), 5(-z), 6(+z).
3523   When the mesh is refined, "Face Sets" contain the new vertices (created during refinement).  Use `DMPlexLabelComplete()` to propagate to coarse-level vertices.
3524 
3525   Developer Notes:
3526   The Gyroid mesh does not currently mark boundary sets.
3527 
3528   References:
3529 . * - Maskery et al, Insights into the mechanical properties of several triply periodic minimal surface lattice structures made by polymer additive manufacturing, 2017.
3530   https://doi.org/10.1016/j.polymer.2017.11.049
3531 
3532 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMSetType()`, `DMCreate()`
3533 @*/
3534 PetscErrorCode DMPlexCreateTPSMesh(MPI_Comm comm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness, DM *dm)
3535 {
3536   PetscFunctionBegin;
3537   PetscCall(DMCreate(comm, dm));
3538   PetscCall(DMSetType(*dm, DMPLEX));
3539   PetscCall(DMPlexCreateTPSMesh_Internal(*dm, tpstype, extent, periodic, tps_distribute, refinements, layers, thickness));
3540   PetscFunctionReturn(PETSC_SUCCESS);
3541 }
3542 
3543 /*@
3544   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
3545 
3546   Collective
3547 
3548   Input Parameters:
3549 + comm    - The communicator for the `DM` object
3550 . dim     - The dimension
3551 . simplex - Use simplices, or tensor product cells
3552 - R       - The radius
3553 
3554   Output Parameter:
3555 . dm - The `DM` object
3556 
3557   Level: beginner
3558 
3559 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBallMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3560 @*/
3561 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscReal R, DM *dm)
3562 {
3563   PetscFunctionBegin;
3564   PetscAssertPointer(dm, 5);
3565   PetscCall(DMCreate(comm, dm));
3566   PetscCall(DMSetType(*dm, DMPLEX));
3567   PetscCall(DMPlexCreateSphereMesh_Internal(*dm, dim, simplex, R));
3568   PetscFunctionReturn(PETSC_SUCCESS);
3569 }
3570 
3571 static PetscErrorCode DMPlexCreateBallMesh_Internal(DM dm, PetscInt dim, PetscReal R)
3572 {
3573   DM      sdm, vol;
3574   DMLabel bdlabel;
3575 
3576   PetscFunctionBegin;
3577   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
3578   PetscCall(DMSetType(sdm, DMPLEX));
3579   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sdm, "bd_"));
3580   PetscCall(DMPlexCreateSphereMesh_Internal(sdm, dim - 1, PETSC_TRUE, R));
3581   PetscCall(DMSetFromOptions(sdm));
3582   PetscCall(DMViewFromOptions(sdm, NULL, "-dm_view"));
3583   PetscCall(DMPlexGenerate(sdm, NULL, PETSC_TRUE, &vol));
3584   PetscCall(DMDestroy(&sdm));
3585   PetscCall(DMPlexReplace_Internal(dm, &vol));
3586   PetscCall(DMCreateLabel(dm, "marker"));
3587   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
3588   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
3589   PetscCall(DMPlexLabelComplete(dm, bdlabel));
3590   PetscFunctionReturn(PETSC_SUCCESS);
3591 }
3592 
3593 /*@
3594   DMPlexCreateBallMesh - Creates a simplex mesh on the d-dimensional ball, B^d.
3595 
3596   Collective
3597 
3598   Input Parameters:
3599 + comm - The communicator for the `DM` object
3600 . dim  - The dimension
3601 - R    - The radius
3602 
3603   Output Parameter:
3604 . dm - The `DM` object
3605 
3606   Options Database Key:
3607 . bd_dm_refine - This will refine the surface mesh preserving the sphere geometry
3608 
3609   Level: beginner
3610 
3611 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3612 @*/
3613 PetscErrorCode DMPlexCreateBallMesh(MPI_Comm comm, PetscInt dim, PetscReal R, DM *dm)
3614 {
3615   PetscFunctionBegin;
3616   PetscCall(DMCreate(comm, dm));
3617   PetscCall(DMSetType(*dm, DMPLEX));
3618   PetscCall(DMPlexCreateBallMesh_Internal(*dm, dim, R));
3619   PetscFunctionReturn(PETSC_SUCCESS);
3620 }
3621 
3622 static PetscErrorCode DMPlexCreateReferenceCell_Internal(DM rdm, DMPolytopeType ct)
3623 {
3624   PetscFunctionBegin;
3625   switch (ct) {
3626   case DM_POLYTOPE_POINT: {
3627     PetscInt    numPoints[1]        = {1};
3628     PetscInt    coneSize[1]         = {0};
3629     PetscInt    cones[1]            = {0};
3630     PetscInt    coneOrientations[1] = {0};
3631     PetscScalar vertexCoords[1]     = {0.0};
3632 
3633     PetscCall(DMSetDimension(rdm, 0));
3634     PetscCall(DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3635   } break;
3636   case DM_POLYTOPE_SEGMENT: {
3637     PetscInt    numPoints[2]        = {2, 1};
3638     PetscInt    coneSize[3]         = {2, 0, 0};
3639     PetscInt    cones[2]            = {1, 2};
3640     PetscInt    coneOrientations[2] = {0, 0};
3641     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3642 
3643     PetscCall(DMSetDimension(rdm, 1));
3644     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3645   } break;
3646   case DM_POLYTOPE_POINT_PRISM_TENSOR: {
3647     PetscInt    numPoints[2]        = {2, 1};
3648     PetscInt    coneSize[3]         = {2, 0, 0};
3649     PetscInt    cones[2]            = {1, 2};
3650     PetscInt    coneOrientations[2] = {0, 0};
3651     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
3652 
3653     PetscCall(DMSetDimension(rdm, 1));
3654     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3655   } break;
3656   case DM_POLYTOPE_TRIANGLE: {
3657     PetscInt    numPoints[2]        = {3, 1};
3658     PetscInt    coneSize[4]         = {3, 0, 0, 0};
3659     PetscInt    cones[3]            = {1, 2, 3};
3660     PetscInt    coneOrientations[3] = {0, 0, 0};
3661     PetscScalar vertexCoords[6]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0};
3662 
3663     PetscCall(DMSetDimension(rdm, 2));
3664     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3665   } break;
3666   case DM_POLYTOPE_QUADRILATERAL: {
3667     PetscInt    numPoints[2]        = {4, 1};
3668     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3669     PetscInt    cones[4]            = {1, 2, 3, 4};
3670     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3671     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0};
3672 
3673     PetscCall(DMSetDimension(rdm, 2));
3674     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3675   } break;
3676   case DM_POLYTOPE_SEG_PRISM_TENSOR: {
3677     PetscInt    numPoints[2]        = {4, 1};
3678     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3679     PetscInt    cones[4]            = {1, 2, 3, 4};
3680     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3681     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0};
3682 
3683     PetscCall(DMSetDimension(rdm, 2));
3684     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3685   } break;
3686   case DM_POLYTOPE_TETRAHEDRON: {
3687     PetscInt    numPoints[2]        = {4, 1};
3688     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
3689     PetscInt    cones[4]            = {1, 2, 3, 4};
3690     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
3691     PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0};
3692 
3693     PetscCall(DMSetDimension(rdm, 3));
3694     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3695   } break;
3696   case DM_POLYTOPE_HEXAHEDRON: {
3697     PetscInt    numPoints[2]        = {8, 1};
3698     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3699     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3700     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3701     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3702 
3703     PetscCall(DMSetDimension(rdm, 3));
3704     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3705   } break;
3706   case DM_POLYTOPE_TRI_PRISM: {
3707     PetscInt    numPoints[2]        = {6, 1};
3708     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3709     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3710     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3711     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3712 
3713     PetscCall(DMSetDimension(rdm, 3));
3714     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3715   } break;
3716   case DM_POLYTOPE_TRI_PRISM_TENSOR: {
3717     PetscInt    numPoints[2]        = {6, 1};
3718     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
3719     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
3720     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
3721     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
3722 
3723     PetscCall(DMSetDimension(rdm, 3));
3724     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3725   } break;
3726   case DM_POLYTOPE_QUAD_PRISM_TENSOR: {
3727     PetscInt    numPoints[2]        = {8, 1};
3728     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
3729     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
3730     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3731     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
3732 
3733     PetscCall(DMSetDimension(rdm, 3));
3734     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3735   } break;
3736   case DM_POLYTOPE_PYRAMID: {
3737     PetscInt    numPoints[2]        = {5, 1};
3738     PetscInt    coneSize[6]         = {5, 0, 0, 0, 0, 0};
3739     PetscInt    cones[5]            = {1, 2, 3, 4, 5};
3740     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
3741     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, 0.0, 0.0, 1.0};
3742 
3743     PetscCall(DMSetDimension(rdm, 3));
3744     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
3745   } break;
3746   default:
3747     SETERRQ(PetscObjectComm((PetscObject)rdm), PETSC_ERR_ARG_WRONG, "Cannot create reference cell for cell type %s", DMPolytopeTypes[ct]);
3748   }
3749   {
3750     PetscInt Nv, v;
3751 
3752     /* Must create the celltype label here so that we do not automatically try to compute the types */
3753     PetscCall(DMCreateLabel(rdm, "celltype"));
3754     PetscCall(DMPlexSetCellType(rdm, 0, ct));
3755     PetscCall(DMPlexGetChart(rdm, NULL, &Nv));
3756     for (v = 1; v < Nv; ++v) PetscCall(DMPlexSetCellType(rdm, v, DM_POLYTOPE_POINT));
3757   }
3758   PetscCall(DMPlexInterpolateInPlace_Internal(rdm));
3759   PetscCall(PetscObjectSetName((PetscObject)rdm, DMPolytopeTypes[ct]));
3760   PetscFunctionReturn(PETSC_SUCCESS);
3761 }
3762 
3763 /*@
3764   DMPlexCreateReferenceCell - Create a `DMPLEX` with the appropriate FEM reference cell
3765 
3766   Collective
3767 
3768   Input Parameters:
3769 + comm - The communicator
3770 - ct   - The cell type of the reference cell
3771 
3772   Output Parameter:
3773 . refdm - The reference cell
3774 
3775   Level: intermediate
3776 
3777 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBoxMesh()`
3778 @*/
3779 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, DMPolytopeType ct, DM *refdm)
3780 {
3781   PetscFunctionBegin;
3782   PetscCall(DMCreate(comm, refdm));
3783   PetscCall(DMSetType(*refdm, DMPLEX));
3784   PetscCall(DMPlexCreateReferenceCell_Internal(*refdm, ct));
3785   PetscFunctionReturn(PETSC_SUCCESS);
3786 }
3787 
3788 static PetscErrorCode DMPlexCreateBoundaryLabel_Private(DM dm, const char name[])
3789 {
3790   DM        plex;
3791   DMLabel   label;
3792   PetscBool hasLabel;
3793 
3794   PetscFunctionBegin;
3795   PetscCall(DMHasLabel(dm, name, &hasLabel));
3796   if (hasLabel) PetscFunctionReturn(PETSC_SUCCESS);
3797   PetscCall(DMCreateLabel(dm, name));
3798   PetscCall(DMGetLabel(dm, name, &label));
3799   PetscCall(DMConvert(dm, DMPLEX, &plex));
3800   PetscCall(DMPlexMarkBoundaryFaces(plex, 1, label));
3801   PetscCall(DMPlexLabelComplete(plex, label));
3802   PetscCall(DMDestroy(&plex));
3803   PetscFunctionReturn(PETSC_SUCCESS);
3804 }
3805 
3806 /*
3807   We use the last coordinate as the radius, the inner radius is lower[dim-1] and the outer radius is upper[dim-1]. Then we map the first coordinate around the circle.
3808 
3809     (x, y) -> (r, theta) = (x[1], (x[0] - lower[0]) * 2\pi/(upper[0] - lower[0]))
3810 */
3811 static void boxToAnnulus(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
3812 {
3813   const PetscReal low = PetscRealPart(constants[0]);
3814   const PetscReal upp = PetscRealPart(constants[1]);
3815   const PetscReal r   = PetscRealPart(u[1]);
3816   const PetscReal th  = 2. * PETSC_PI * (PetscRealPart(u[0]) - low) / (upp - low);
3817 
3818   f0[0] = r * PetscCosReal(th);
3819   f0[1] = r * PetscSinReal(th);
3820 }
3821 
3822 const char *const DMPlexShapes[] = {"box", "box_surface", "ball", "sphere", "cylinder", "schwarz_p", "gyroid", "doublet", "annulus", "hypercubic", "zbox", "unknown", "DMPlexShape", "DM_SHAPE_", NULL};
3823 
3824 static PetscErrorCode DMPlexCreateFromOptions_Internal(PetscOptionItems *PetscOptionsObject, PetscBool *useCoordSpace, DM dm)
3825 {
3826   DMPlexShape    shape   = DM_SHAPE_BOX;
3827   DMPolytopeType cell    = DM_POLYTOPE_TRIANGLE;
3828   PetscInt       dim     = 2;
3829   PetscBool      simplex = PETSC_TRUE, interpolate = PETSC_TRUE, adjCone = PETSC_FALSE, adjClosure = PETSC_TRUE, refDomain = PETSC_FALSE;
3830   PetscBool      flg, flg2, fflg, bdfflg, nameflg;
3831   MPI_Comm       comm;
3832   char           filename[PETSC_MAX_PATH_LEN]   = "<unspecified>";
3833   char           bdFilename[PETSC_MAX_PATH_LEN] = "<unspecified>";
3834   char           plexname[PETSC_MAX_PATH_LEN]   = "";
3835 
3836   PetscFunctionBegin;
3837   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromOptions, dm, 0, 0, 0));
3838   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3839   /* TODO Turn this into a registration interface */
3840   PetscCall(PetscOptionsString("-dm_plex_filename", "File containing a mesh", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &fflg));
3841   PetscCall(PetscOptionsString("-dm_plex_boundary_filename", "File containing a mesh boundary", "DMPlexCreateFromFile", bdFilename, bdFilename, sizeof(bdFilename), &bdfflg));
3842   PetscCall(PetscOptionsString("-dm_plex_name", "Name of the mesh in the file", "DMPlexCreateFromFile", plexname, plexname, sizeof(plexname), &nameflg));
3843   PetscCall(PetscOptionsEnum("-dm_plex_cell", "Cell shape", "", DMPolytopeTypes, (PetscEnum)cell, (PetscEnum *)&cell, NULL));
3844   PetscCall(PetscOptionsBool("-dm_plex_reference_cell_domain", "Use a reference cell domain", "", refDomain, &refDomain, NULL));
3845   PetscCall(PetscOptionsEnum("-dm_plex_shape", "Shape for built-in mesh", "", DMPlexShapes, (PetscEnum)shape, (PetscEnum *)&shape, &flg));
3846   PetscCall(PetscOptionsBoundedInt("-dm_plex_dim", "Topological dimension of the mesh", "DMGetDimension", dim, &dim, &flg, 0));
3847   PetscCheck(dim >= 0, comm, PETSC_ERR_ARG_OUTOFRANGE, "Dimension %" PetscInt_FMT " should be in [0, infinity)", dim);
3848   PetscCall(PetscOptionsBool("-dm_plex_simplex", "Mesh cell shape", "", simplex, &simplex, &flg));
3849   PetscCall(PetscOptionsBool("-dm_plex_interpolate", "Flag to create edges and faces automatically", "", interpolate, &interpolate, &flg));
3850   PetscCall(PetscOptionsBool("-dm_plex_adj_cone", "Set adjacency direction", "DMSetBasicAdjacency", adjCone, &adjCone, &flg));
3851   PetscCall(PetscOptionsBool("-dm_plex_adj_closure", "Set adjacency size", "DMSetBasicAdjacency", adjClosure, &adjClosure, &flg2));
3852   if (flg || flg2) PetscCall(DMSetBasicAdjacency(dm, adjCone, adjClosure));
3853 
3854   switch (cell) {
3855   case DM_POLYTOPE_POINT:
3856   case DM_POLYTOPE_SEGMENT:
3857   case DM_POLYTOPE_POINT_PRISM_TENSOR:
3858   case DM_POLYTOPE_TRIANGLE:
3859   case DM_POLYTOPE_QUADRILATERAL:
3860   case DM_POLYTOPE_TETRAHEDRON:
3861   case DM_POLYTOPE_HEXAHEDRON:
3862     *useCoordSpace = PETSC_TRUE;
3863     break;
3864   default:
3865     *useCoordSpace = PETSC_FALSE;
3866     break;
3867   }
3868 
3869   if (fflg) {
3870     DM dmnew;
3871 
3872     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), filename, plexname, interpolate, &dmnew));
3873     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3874     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3875   } else if (refDomain) {
3876     PetscCall(DMPlexCreateReferenceCell_Internal(dm, cell));
3877   } else if (bdfflg) {
3878     DM bdm, dmnew;
3879 
3880     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), bdFilename, plexname, interpolate, &bdm));
3881     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)bdm, "bd_"));
3882     PetscCall(DMSetFromOptions(bdm));
3883     PetscCall(DMPlexGenerate(bdm, NULL, interpolate, &dmnew));
3884     PetscCall(DMDestroy(&bdm));
3885     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
3886     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
3887   } else {
3888     PetscCall(PetscObjectSetName((PetscObject)dm, DMPlexShapes[shape]));
3889     switch (shape) {
3890     case DM_SHAPE_BOX:
3891     case DM_SHAPE_ZBOX:
3892     case DM_SHAPE_ANNULUS: {
3893       PetscInt       faces[3]  = {0, 0, 0};
3894       PetscReal      lower[3]  = {0, 0, 0};
3895       PetscReal      upper[3]  = {1, 1, 1};
3896       DMBoundaryType bdt[3]    = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3897       PetscBool      isAnnular = shape == DM_SHAPE_ANNULUS ? PETSC_TRUE : PETSC_FALSE;
3898       PetscInt       i, n;
3899 
3900       n = dim;
3901       for (i = 0; i < dim; ++i) faces[i] = (dim == 1 ? 1 : 4 - dim);
3902       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3903       n = 3;
3904       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3905       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3906       n = 3;
3907       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3908       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3909       n = 3;
3910       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
3911       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
3912 
3913       PetscCheck(!isAnnular || dim == 2, comm, PETSC_ERR_ARG_OUTOFRANGE, "Only two dimensional annuli have been implemented");
3914       if (isAnnular)
3915         for (i = 0; i < dim - 1; ++i) bdt[i] = DM_BOUNDARY_PERIODIC;
3916 
3917       switch (cell) {
3918       case DM_POLYTOPE_TRI_PRISM_TENSOR:
3919         PetscCall(DMPlexCreateWedgeBoxMesh_Internal(dm, faces, lower, upper, bdt));
3920         if (!interpolate) {
3921           DM udm;
3922 
3923           PetscCall(DMPlexUninterpolate(dm, &udm));
3924           PetscCall(DMPlexReplace_Internal(dm, &udm));
3925         }
3926         break;
3927       default:
3928         PetscCall(DMPlexCreateBoxMesh_Internal(dm, shape, dim, simplex, faces, lower, upper, bdt, interpolate));
3929         break;
3930       }
3931       if (isAnnular) {
3932         DM          cdm;
3933         PetscDS     cds;
3934         PetscScalar bounds[2] = {lower[0], upper[0]};
3935 
3936         // Fix coordinates for annular region
3937         PetscCall(DMSetPeriodicity(dm, NULL, NULL, NULL));
3938         PetscCall(DMSetCellCoordinatesLocal(dm, NULL));
3939         PetscCall(DMSetCellCoordinates(dm, NULL));
3940         PetscCall(DMPlexCreateCoordinateSpace(dm, 1, NULL));
3941         PetscCall(DMGetCoordinateDM(dm, &cdm));
3942         PetscCall(DMGetDS(cdm, &cds));
3943         PetscCall(PetscDSSetConstants(cds, 2, bounds));
3944         PetscCall(DMPlexRemapGeometry(dm, 0.0, boxToAnnulus));
3945       }
3946     } break;
3947     case DM_SHAPE_BOX_SURFACE: {
3948       PetscInt  faces[3] = {0, 0, 0};
3949       PetscReal lower[3] = {0, 0, 0};
3950       PetscReal upper[3] = {1, 1, 1};
3951       PetscInt  i, n;
3952 
3953       n = dim + 1;
3954       for (i = 0; i < dim + 1; ++i) faces[i] = (dim + 1 == 1 ? 1 : 4 - (dim + 1));
3955       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
3956       n = 3;
3957       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
3958       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3959       n = 3;
3960       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
3961       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
3962       PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(dm, dim + 1, faces, lower, upper, interpolate));
3963     } break;
3964     case DM_SHAPE_SPHERE: {
3965       PetscReal R = 1.0;
3966 
3967       PetscCall(PetscOptionsReal("-dm_plex_sphere_radius", "Radius of the sphere", "", R, &R, &flg));
3968       PetscCall(DMPlexCreateSphereMesh_Internal(dm, dim, simplex, R));
3969     } break;
3970     case DM_SHAPE_BALL: {
3971       PetscReal R = 1.0;
3972 
3973       PetscCall(PetscOptionsReal("-dm_plex_ball_radius", "Radius of the ball", "", R, &R, &flg));
3974       PetscCall(DMPlexCreateBallMesh_Internal(dm, dim, R));
3975     } break;
3976     case DM_SHAPE_CYLINDER: {
3977       DMBoundaryType bdt = DM_BOUNDARY_NONE;
3978       PetscInt       Nw  = 6;
3979 
3980       PetscCall(PetscOptionsEnum("-dm_plex_cylinder_bd", "Boundary type in the z direction", "", DMBoundaryTypes, (PetscEnum)bdt, (PetscEnum *)&bdt, NULL));
3981       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_wedges", "Number of wedges around the cylinder", "", Nw, &Nw, NULL));
3982       switch (cell) {
3983       case DM_POLYTOPE_TRI_PRISM_TENSOR:
3984         PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(dm, Nw, interpolate));
3985         break;
3986       default:
3987         PetscCall(DMPlexCreateHexCylinderMesh_Internal(dm, bdt));
3988         break;
3989       }
3990     } break;
3991     case DM_SHAPE_SCHWARZ_P: // fallthrough
3992     case DM_SHAPE_GYROID: {
3993       PetscInt       extent[3] = {1, 1, 1}, refine = 0, layers = 0, three;
3994       PetscReal      thickness   = 0.;
3995       DMBoundaryType periodic[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
3996       DMPlexTPSType  tps_type    = shape == DM_SHAPE_SCHWARZ_P ? DMPLEX_TPS_SCHWARZ_P : DMPLEX_TPS_GYROID;
3997       PetscBool      tps_distribute;
3998       PetscCall(PetscOptionsIntArray("-dm_plex_tps_extent", "Number of replicas for each of three dimensions", NULL, extent, (three = 3, &three), NULL));
3999       PetscCall(PetscOptionsInt("-dm_plex_tps_refine", "Number of refinements", NULL, refine, &refine, NULL));
4000       PetscCall(PetscOptionsEnumArray("-dm_plex_tps_periodic", "Periodicity in each of three dimensions", NULL, DMBoundaryTypes, (PetscEnum *)periodic, (three = 3, &three), NULL));
4001       PetscCall(PetscOptionsInt("-dm_plex_tps_layers", "Number of layers in volumetric extrusion (or zero to not extrude)", NULL, layers, &layers, NULL));
4002       PetscCall(PetscOptionsReal("-dm_plex_tps_thickness", "Thickness of volumetric extrusion", NULL, thickness, &thickness, NULL));
4003       PetscCall(DMPlexDistributeGetDefault(dm, &tps_distribute));
4004       PetscCall(PetscOptionsBool("-dm_plex_tps_distribute", "Distribute the 2D mesh prior to refinement and extrusion", NULL, tps_distribute, &tps_distribute, NULL));
4005       PetscCall(DMPlexCreateTPSMesh_Internal(dm, tps_type, extent, periodic, tps_distribute, refine, layers, thickness));
4006     } break;
4007     case DM_SHAPE_DOUBLET: {
4008       DM        dmnew;
4009       PetscReal rl = 0.0;
4010 
4011       PetscCall(PetscOptionsReal("-dm_plex_doublet_refinementlimit", "Refinement limit", NULL, rl, &rl, NULL));
4012       PetscCall(DMPlexCreateDoublet(PetscObjectComm((PetscObject)dm), dim, simplex, interpolate, rl, &dmnew));
4013       PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4014       PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4015     } break;
4016     case DM_SHAPE_HYPERCUBIC: {
4017       PetscInt       *edges;
4018       PetscReal      *lower, *upper;
4019       DMBoundaryType *bdt;
4020       PetscInt        n, d;
4021 
4022       *useCoordSpace = PETSC_FALSE;
4023       PetscCall(PetscMalloc4(dim, &edges, dim, &lower, dim, &upper, dim, &bdt));
4024       for (d = 0; d < dim; ++d) {
4025         edges[d] = 1;
4026         lower[d] = 0.;
4027         upper[d] = 1.;
4028         bdt[d]   = DM_BOUNDARY_PERIODIC;
4029       }
4030       n = dim;
4031       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", edges, &n, &flg));
4032       n = dim;
4033       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
4034       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4035       n = dim;
4036       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
4037       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4038       n = dim;
4039       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
4040       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4041       PetscCall(DMPlexCreateHypercubicMesh_Internal(dm, dim, lower, upper, edges, bdt));
4042       PetscCall(PetscFree4(edges, lower, upper, bdt));
4043     } break;
4044     default:
4045       SETERRQ(comm, PETSC_ERR_SUP, "Domain shape %s is unsupported", DMPlexShapes[shape]);
4046     }
4047   }
4048   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
4049   if (!((PetscObject)dm)->name && nameflg) PetscCall(PetscObjectSetName((PetscObject)dm, plexname));
4050   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromOptions, dm, 0, 0, 0));
4051   PetscFunctionReturn(PETSC_SUCCESS);
4052 }
4053 
4054 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
4055 {
4056   DM_Plex  *mesh = (DM_Plex *)dm->data;
4057   PetscBool flg, flg2;
4058   char      bdLabel[PETSC_MAX_PATH_LEN];
4059 
4060   PetscFunctionBegin;
4061   /* Handle viewing */
4062   PetscCall(PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL));
4063   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fem", "Debug output level all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL, 0));
4064   PetscCall(PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL));
4065   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL, 0));
4066   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_locate", "Debug output level all point location computations", "DMLocatePoints", 0, &mesh->printLocate, NULL, 0));
4067   PetscCall(DMMonitorSetFromOptions(dm, "-dm_plex_monitor_throughput", "Monitor the simulation throughput", "DMPlexMonitorThroughput", DMPlexMonitorThroughput, NULL, &flg));
4068   if (flg) PetscCall(PetscLogDefaultBegin());
4069   /* Labeling */
4070   PetscCall(PetscOptionsString("-dm_plex_boundary_label", "Label to mark the mesh boundary", "", bdLabel, bdLabel, sizeof(bdLabel), &flg));
4071   if (flg) PetscCall(DMPlexCreateBoundaryLabel_Private(dm, bdLabel));
4072   /* Point Location */
4073   PetscCall(PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL));
4074   /* Partitioning and distribution */
4075   PetscCall(PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL));
4076   /* Generation and remeshing */
4077   PetscCall(PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL));
4078   /* Projection behavior */
4079   PetscCall(PetscOptionsBoundedInt("-dm_plex_max_projection_height", "Maximum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL, 0));
4080   PetscCall(PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL));
4081   /* Checking structure */
4082   {
4083     PetscBool all = PETSC_FALSE;
4084 
4085     PetscCall(PetscOptionsBool("-dm_plex_check_all", "Perform all basic checks", "DMPlexCheck", PETSC_FALSE, &all, NULL));
4086     if (all) {
4087       PetscCall(DMPlexCheck(dm));
4088     } else {
4089       PetscCall(PetscOptionsBool("-dm_plex_check_symmetry", "Check that the adjacency information in the mesh is symmetric", "DMPlexCheckSymmetry", PETSC_FALSE, &flg, &flg2));
4090       if (flg && flg2) PetscCall(DMPlexCheckSymmetry(dm));
4091       PetscCall(PetscOptionsBool("-dm_plex_check_skeleton", "Check that each cell has the correct number of vertices (only for homogeneous simplex or tensor meshes)", "DMPlexCheckSkeleton", PETSC_FALSE, &flg, &flg2));
4092       if (flg && flg2) PetscCall(DMPlexCheckSkeleton(dm, 0));
4093       PetscCall(PetscOptionsBool("-dm_plex_check_faces", "Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type", "DMPlexCheckFaces", PETSC_FALSE, &flg, &flg2));
4094       if (flg && flg2) PetscCall(DMPlexCheckFaces(dm, 0));
4095       PetscCall(PetscOptionsBool("-dm_plex_check_geometry", "Check that cells have positive volume", "DMPlexCheckGeometry", PETSC_FALSE, &flg, &flg2));
4096       if (flg && flg2) PetscCall(DMPlexCheckGeometry(dm));
4097       PetscCall(PetscOptionsBool("-dm_plex_check_pointsf", "Check some necessary conditions for PointSF", "DMPlexCheckPointSF", PETSC_FALSE, &flg, &flg2));
4098       if (flg && flg2) PetscCall(DMPlexCheckPointSF(dm, NULL, PETSC_FALSE));
4099       PetscCall(PetscOptionsBool("-dm_plex_check_interface_cones", "Check points on inter-partition interfaces have conforming order of cone points", "DMPlexCheckInterfaceCones", PETSC_FALSE, &flg, &flg2));
4100       if (flg && flg2) PetscCall(DMPlexCheckInterfaceCones(dm));
4101     }
4102     PetscCall(PetscOptionsBool("-dm_plex_check_cell_shape", "Check cell shape", "DMPlexCheckCellShape", PETSC_FALSE, &flg, &flg2));
4103     if (flg && flg2) PetscCall(DMPlexCheckCellShape(dm, PETSC_TRUE, PETSC_DETERMINE));
4104   }
4105   {
4106     PetscReal scale = 1.0;
4107 
4108     PetscCall(PetscOptionsReal("-dm_plex_scale", "Scale factor for mesh coordinates", "DMPlexScale", scale, &scale, &flg));
4109     if (flg) {
4110       Vec coordinates, coordinatesLocal;
4111 
4112       PetscCall(DMGetCoordinates(dm, &coordinates));
4113       PetscCall(DMGetCoordinatesLocal(dm, &coordinatesLocal));
4114       PetscCall(VecScale(coordinates, scale));
4115       PetscCall(VecScale(coordinatesLocal, scale));
4116     }
4117   }
4118   PetscCall(PetscPartitionerSetFromOptions(mesh->partitioner));
4119   PetscFunctionReturn(PETSC_SUCCESS);
4120 }
4121 
4122 PetscErrorCode DMSetFromOptions_Overlap_Plex(DM dm, PetscOptionItems *PetscOptionsObject, PetscInt *overlap)
4123 {
4124   PetscInt  numOvLabels = 16, numOvExLabels = 16;
4125   char     *ovLabelNames[16], *ovExLabelNames[16];
4126   PetscInt  numOvValues = 16, numOvExValues = 16, l;
4127   PetscBool flg;
4128 
4129   PetscFunctionBegin;
4130   PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", *overlap, overlap, NULL, 0));
4131   PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_labels", "List of overlap label names", "DMPlexDistribute", ovLabelNames, &numOvLabels, &flg));
4132   if (!flg) numOvLabels = 0;
4133   if (numOvLabels) {
4134     ((DM_Plex *)dm->data)->numOvLabels = numOvLabels;
4135     for (l = 0; l < numOvLabels; ++l) {
4136       PetscCall(DMGetLabel(dm, ovLabelNames[l], &((DM_Plex *)dm->data)->ovLabels[l]));
4137       PetscCheck(((DM_Plex *)dm->data)->ovLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovLabelNames[l]);
4138       PetscCall(PetscFree(ovLabelNames[l]));
4139     }
4140     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_values", "List of overlap label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovValues, &numOvValues, &flg));
4141     if (!flg) numOvValues = 0;
4142     PetscCheck(numOvLabels == numOvValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvLabels, numOvValues);
4143 
4144     PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_exclude_labels", "List of overlap exclude label names", "DMPlexDistribute", ovExLabelNames, &numOvExLabels, &flg));
4145     if (!flg) numOvExLabels = 0;
4146     ((DM_Plex *)dm->data)->numOvExLabels = numOvExLabels;
4147     for (l = 0; l < numOvExLabels; ++l) {
4148       PetscCall(DMGetLabel(dm, ovExLabelNames[l], &((DM_Plex *)dm->data)->ovExLabels[l]));
4149       PetscCheck(((DM_Plex *)dm->data)->ovExLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovExLabelNames[l]);
4150       PetscCall(PetscFree(ovExLabelNames[l]));
4151     }
4152     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_exclude_values", "List of overlap exclude label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovExValues, &numOvExValues, &flg));
4153     if (!flg) numOvExValues = 0;
4154     PetscCheck(numOvExLabels == numOvExValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of exclude labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvExLabels, numOvExValues);
4155   }
4156   PetscFunctionReturn(PETSC_SUCCESS);
4157 }
4158 
4159 static PetscErrorCode DMSetFromOptions_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
4160 {
4161   PetscFunctionList        ordlist;
4162   char                     oname[256];
4163   PetscReal                volume    = -1.0;
4164   PetscInt                 prerefine = 0, refine = 0, r, coarsen = 0, overlap = 0, extLayers = 0, dim;
4165   PetscBool                uniformOrig, created = PETSC_FALSE, uniform = PETSC_TRUE, distribute, interpolate = PETSC_TRUE, coordSpace = PETSC_TRUE, remap = PETSC_TRUE, ghostCells = PETSC_FALSE, isHierarchy, ignoreModel = PETSC_FALSE, flg;
4166   DMPlexReorderDefaultFlag reorder;
4167 
4168   PetscFunctionBegin;
4169   PetscOptionsHeadBegin(PetscOptionsObject, "DMPlex Options");
4170   if (dm->cloneOpts) goto non_refine;
4171   /* Handle automatic creation */
4172   PetscCall(DMGetDimension(dm, &dim));
4173   if (dim < 0) {
4174     PetscCall(DMPlexCreateFromOptions_Internal(PetscOptionsObject, &coordSpace, dm));
4175     created = PETSC_TRUE;
4176   }
4177   PetscCall(DMGetDimension(dm, &dim));
4178   /* Handle interpolation before distribution */
4179   PetscCall(PetscOptionsBool("-dm_plex_interpolate_pre", "Flag to interpolate mesh before distribution", "", interpolate, &interpolate, &flg));
4180   if (flg) {
4181     DMPlexInterpolatedFlag interpolated;
4182 
4183     PetscCall(DMPlexIsInterpolated(dm, &interpolated));
4184     if (interpolated == DMPLEX_INTERPOLATED_FULL && !interpolate) {
4185       DM udm;
4186 
4187       PetscCall(DMPlexUninterpolate(dm, &udm));
4188       PetscCall(DMPlexReplace_Internal(dm, &udm));
4189     } else if (interpolated != DMPLEX_INTERPOLATED_FULL && interpolate) {
4190       DM idm;
4191 
4192       PetscCall(DMPlexInterpolate(dm, &idm));
4193       PetscCall(DMPlexReplace_Internal(dm, &idm));
4194     }
4195   }
4196   /* Handle DMPlex refinement before distribution */
4197   PetscCall(PetscOptionsBool("-dm_refine_ignore_model", "Flag to ignore the geometry model when refining", "DMCreate", ignoreModel, &ignoreModel, &flg));
4198   if (flg) ((DM_Plex *)dm->data)->ignoreModel = ignoreModel;
4199   PetscCall(DMPlexGetRefinementUniform(dm, &uniformOrig));
4200   PetscCall(PetscOptionsBoundedInt("-dm_refine_pre", "The number of refinements before distribution", "DMCreate", prerefine, &prerefine, NULL, 0));
4201   PetscCall(PetscOptionsBool("-dm_refine_remap_pre", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
4202   PetscCall(PetscOptionsBool("-dm_refine_uniform_pre", "Flag for uniform refinement before distribution", "DMCreate", uniform, &uniform, &flg));
4203   if (flg) PetscCall(DMPlexSetRefinementUniform(dm, uniform));
4204   PetscCall(PetscOptionsReal("-dm_refine_volume_limit_pre", "The maximum cell volume after refinement before distribution", "DMCreate", volume, &volume, &flg));
4205   if (flg) {
4206     PetscCall(DMPlexSetRefinementUniform(dm, PETSC_FALSE));
4207     PetscCall(DMPlexSetRefinementLimit(dm, volume));
4208     prerefine = PetscMax(prerefine, 1);
4209   }
4210   for (r = 0; r < prerefine; ++r) {
4211     DM             rdm;
4212     PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4213 
4214     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4215     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
4216     PetscCall(DMPlexReplace_Internal(dm, &rdm));
4217     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4218     if (coordFunc && remap) {
4219       PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4220       ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4221     }
4222   }
4223   PetscCall(DMPlexSetRefinementUniform(dm, uniformOrig));
4224   /* Handle DMPlex extrusion before distribution */
4225   PetscCall(PetscOptionsBoundedInt("-dm_extrude", "The number of layers to extrude", "", extLayers, &extLayers, NULL, 0));
4226   if (extLayers) {
4227     DM edm;
4228 
4229     PetscCall(DMExtrude(dm, extLayers, &edm));
4230     PetscCall(DMPlexReplace_Internal(dm, &edm));
4231     ((DM_Plex *)dm->data)->coordFunc = NULL;
4232     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4233     extLayers = 0;
4234     PetscCall(DMGetDimension(dm, &dim));
4235   }
4236   /* Handle DMPlex reordering before distribution */
4237   PetscCall(DMPlexReorderGetDefault(dm, &reorder));
4238   PetscCall(MatGetOrderingList(&ordlist));
4239   PetscCall(PetscStrncpy(oname, MATORDERINGNATURAL, sizeof(oname)));
4240   PetscCall(PetscOptionsFList("-dm_plex_reorder", "Set mesh reordering type", "DMPlexGetOrdering", ordlist, MATORDERINGNATURAL, oname, sizeof(oname), &flg));
4241   if (reorder == DMPLEX_REORDER_DEFAULT_TRUE || flg) {
4242     DM pdm;
4243     IS perm;
4244 
4245     PetscCall(DMPlexGetOrdering(dm, oname, NULL, &perm));
4246     PetscCall(DMPlexPermute(dm, perm, &pdm));
4247     PetscCall(ISDestroy(&perm));
4248     PetscCall(DMPlexReplace_Internal(dm, &pdm));
4249     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4250   }
4251   /* Handle DMPlex distribution */
4252   PetscCall(DMPlexDistributeGetDefault(dm, &distribute));
4253   PetscCall(PetscOptionsBool("-dm_distribute", "Flag to redistribute a mesh among processes", "DMPlexDistribute", distribute, &distribute, NULL));
4254   PetscCall(DMSetFromOptions_Overlap_Plex(dm, PetscOptionsObject, &overlap));
4255   if (distribute) {
4256     DM               pdm = NULL;
4257     PetscPartitioner part;
4258 
4259     PetscCall(DMPlexGetPartitioner(dm, &part));
4260     PetscCall(PetscPartitionerSetFromOptions(part));
4261     PetscCall(DMPlexDistribute(dm, overlap, NULL, &pdm));
4262     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
4263   }
4264   /* Create coordinate space */
4265   if (created) {
4266     DM_Plex  *mesh   = (DM_Plex *)dm->data;
4267     PetscInt  degree = 1;
4268     PetscInt  height = 0;
4269     DM        cdm;
4270     PetscBool flg;
4271 
4272     PetscCall(PetscOptionsBool("-dm_coord_space", "Use an FEM space for coordinates", "", coordSpace, &coordSpace, &flg));
4273     PetscCall(PetscOptionsInt("-dm_coord_petscspace_degree", "FEM degree for coordinate space", "", degree, &degree, NULL));
4274     if (coordSpace) PetscCall(DMPlexCreateCoordinateSpace(dm, degree, mesh->coordFunc));
4275     PetscCall(DMGetCoordinateDM(dm, &cdm));
4276     if (flg && !coordSpace) {
4277       PetscDS      cds;
4278       PetscObject  obj;
4279       PetscClassId id;
4280 
4281       PetscCall(DMGetDS(cdm, &cds));
4282       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
4283       PetscCall(PetscObjectGetClassId(obj, &id));
4284       if (id == PETSCFE_CLASSID) {
4285         PetscContainer dummy;
4286 
4287         PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &dummy));
4288         PetscCall(PetscObjectSetName((PetscObject)dummy, "coordinates"));
4289         PetscCall(DMSetField(cdm, 0, NULL, (PetscObject)dummy));
4290         PetscCall(PetscContainerDestroy(&dummy));
4291         PetscCall(DMClearDS(cdm));
4292       }
4293       mesh->coordFunc = NULL;
4294     }
4295     PetscCall(PetscOptionsBool("-dm_sparse_localize", "Localize only necessary cells", "", dm->sparseLocalize, &dm->sparseLocalize, &flg));
4296     PetscCall(PetscOptionsInt("-dm_localize_height", "Localize edges and faces in addition to cells", "", height, &height, &flg));
4297     if (flg) PetscCall(DMPlexSetMaxProjectionHeight(cdm, height));
4298     PetscCall(DMLocalizeCoordinates(dm));
4299   }
4300   /* Handle DMPlex refinement */
4301   remap = PETSC_TRUE;
4302   PetscCall(PetscOptionsBoundedInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL, 0));
4303   PetscCall(PetscOptionsBool("-dm_refine_remap", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
4304   PetscCall(PetscOptionsBoundedInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy, 0));
4305   if (refine) PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
4306   if (refine && isHierarchy) {
4307     DM *dms, coarseDM;
4308 
4309     PetscCall(DMGetCoarseDM(dm, &coarseDM));
4310     PetscCall(PetscObjectReference((PetscObject)coarseDM));
4311     PetscCall(PetscMalloc1(refine, &dms));
4312     PetscCall(DMRefineHierarchy(dm, refine, dms));
4313     /* Total hack since we do not pass in a pointer */
4314     PetscCall(DMPlexSwap_Static(dm, dms[refine - 1]));
4315     if (refine == 1) {
4316       PetscCall(DMSetCoarseDM(dm, dms[0]));
4317       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4318     } else {
4319       PetscCall(DMSetCoarseDM(dm, dms[refine - 2]));
4320       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4321       PetscCall(DMSetCoarseDM(dms[0], dms[refine - 1]));
4322       PetscCall(DMPlexSetRegularRefinement(dms[0], PETSC_TRUE));
4323     }
4324     PetscCall(DMSetCoarseDM(dms[refine - 1], coarseDM));
4325     PetscCall(PetscObjectDereference((PetscObject)coarseDM));
4326     /* Free DMs */
4327     for (r = 0; r < refine; ++r) {
4328       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4329       PetscCall(DMDestroy(&dms[r]));
4330     }
4331     PetscCall(PetscFree(dms));
4332   } else {
4333     for (r = 0; r < refine; ++r) {
4334       DM             rdm;
4335       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4336 
4337       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4338       PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
4339       /* Total hack since we do not pass in a pointer */
4340       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4341       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4342       if (coordFunc && remap) {
4343         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4344         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4345       }
4346     }
4347   }
4348   /* Handle DMPlex coarsening */
4349   PetscCall(PetscOptionsBoundedInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL, 0));
4350   PetscCall(PetscOptionsBoundedInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy, 0));
4351   if (coarsen && isHierarchy) {
4352     DM *dms;
4353 
4354     PetscCall(PetscMalloc1(coarsen, &dms));
4355     PetscCall(DMCoarsenHierarchy(dm, coarsen, dms));
4356     /* Free DMs */
4357     for (r = 0; r < coarsen; ++r) {
4358       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4359       PetscCall(DMDestroy(&dms[r]));
4360     }
4361     PetscCall(PetscFree(dms));
4362   } else {
4363     for (r = 0; r < coarsen; ++r) {
4364       DM             cdm;
4365       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4366 
4367       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4368       PetscCall(DMCoarsen(dm, PetscObjectComm((PetscObject)dm), &cdm));
4369       /* Total hack since we do not pass in a pointer */
4370       PetscCall(DMPlexReplace_Internal(dm, &cdm));
4371       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4372       if (coordFunc) {
4373         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4374         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4375       }
4376     }
4377   }
4378   /* Handle ghost cells */
4379   PetscCall(PetscOptionsBool("-dm_plex_create_fv_ghost_cells", "Flag to create finite volume ghost cells on the boundary", "DMCreate", ghostCells, &ghostCells, NULL));
4380   if (ghostCells) {
4381     DM   gdm;
4382     char lname[PETSC_MAX_PATH_LEN];
4383 
4384     lname[0] = '\0';
4385     PetscCall(PetscOptionsString("-dm_plex_fv_ghost_cells_label", "Label name for ghost cells boundary", "DMCreate", lname, lname, sizeof(lname), &flg));
4386     PetscCall(DMPlexConstructGhostCells(dm, flg ? lname : NULL, NULL, &gdm));
4387     PetscCall(DMPlexReplace_Internal(dm, &gdm));
4388   }
4389   /* Handle 1D order */
4390   if (reorder != DMPLEX_REORDER_DEFAULT_FALSE && dim == 1) {
4391     DM           cdm, rdm;
4392     PetscDS      cds;
4393     PetscObject  obj;
4394     PetscClassId id = PETSC_OBJECT_CLASSID;
4395     IS           perm;
4396     PetscInt     Nf;
4397     PetscBool    distributed;
4398 
4399     PetscCall(DMPlexIsDistributed(dm, &distributed));
4400     PetscCall(DMGetCoordinateDM(dm, &cdm));
4401     PetscCall(DMGetDS(cdm, &cds));
4402     PetscCall(PetscDSGetNumFields(cds, &Nf));
4403     if (Nf) {
4404       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
4405       PetscCall(PetscObjectGetClassId(obj, &id));
4406     }
4407     if (!distributed && id != PETSCFE_CLASSID) {
4408       PetscCall(DMPlexGetOrdering1D(dm, &perm));
4409       PetscCall(DMPlexPermute(dm, perm, &rdm));
4410       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4411       PetscCall(ISDestroy(&perm));
4412     }
4413   }
4414 /* Handle */
4415 non_refine:
4416   PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4417   PetscOptionsHeadEnd();
4418   PetscFunctionReturn(PETSC_SUCCESS);
4419 }
4420 
4421 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm, Vec *vec)
4422 {
4423   PetscFunctionBegin;
4424   PetscCall(DMCreateGlobalVector_Section_Private(dm, vec));
4425   /* PetscCall(VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM)); */
4426   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex));
4427   PetscCall(VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void))VecView_Plex_Native));
4428   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex));
4429   PetscCall(VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void))VecLoad_Plex_Native));
4430   PetscFunctionReturn(PETSC_SUCCESS);
4431 }
4432 
4433 static PetscErrorCode DMCreateLocalVector_Plex(DM dm, Vec *vec)
4434 {
4435   PetscFunctionBegin;
4436   PetscCall(DMCreateLocalVector_Section_Private(dm, vec));
4437   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex_Local));
4438   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex_Local));
4439   PetscFunctionReturn(PETSC_SUCCESS);
4440 }
4441 
4442 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
4443 {
4444   PetscInt depth, d;
4445 
4446   PetscFunctionBegin;
4447   PetscCall(DMPlexGetDepth(dm, &depth));
4448   if (depth == 1) {
4449     PetscCall(DMGetDimension(dm, &d));
4450     if (dim == 0) PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4451     else if (dim == d) PetscCall(DMPlexGetDepthStratum(dm, 1, pStart, pEnd));
4452     else {
4453       *pStart = 0;
4454       *pEnd   = 0;
4455     }
4456   } else {
4457     PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
4458   }
4459   PetscFunctionReturn(PETSC_SUCCESS);
4460 }
4461 
4462 static PetscErrorCode DMGetNeighbors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
4463 {
4464   PetscSF            sf;
4465   PetscInt           niranks, njranks, n;
4466   const PetscMPIInt *iranks, *jranks;
4467   DM_Plex           *data = (DM_Plex *)dm->data;
4468 
4469   PetscFunctionBegin;
4470   PetscCall(DMGetPointSF(dm, &sf));
4471   if (!data->neighbors) {
4472     PetscCall(PetscSFSetUp(sf));
4473     PetscCall(PetscSFGetRootRanks(sf, &njranks, &jranks, NULL, NULL, NULL));
4474     PetscCall(PetscSFGetLeafRanks(sf, &niranks, &iranks, NULL, NULL));
4475     PetscCall(PetscMalloc1(njranks + niranks + 1, &data->neighbors));
4476     PetscCall(PetscArraycpy(data->neighbors + 1, jranks, njranks));
4477     PetscCall(PetscArraycpy(data->neighbors + njranks + 1, iranks, niranks));
4478     n = njranks + niranks;
4479     PetscCall(PetscSortRemoveDupsMPIInt(&n, data->neighbors + 1));
4480     /* The following cast should never fail: can't have more neighbors than PETSC_MPI_INT_MAX */
4481     PetscCall(PetscMPIIntCast(n, data->neighbors));
4482   }
4483   if (nranks) *nranks = data->neighbors[0];
4484   if (ranks) {
4485     if (data->neighbors[0]) *ranks = data->neighbors + 1;
4486     else *ranks = NULL;
4487   }
4488   PetscFunctionReturn(PETSC_SUCCESS);
4489 }
4490 
4491 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM, DM, Mat, Vec, Vec);
4492 
4493 static PetscErrorCode DMInitialize_Plex(DM dm)
4494 {
4495   PetscFunctionBegin;
4496   dm->ops->view                      = DMView_Plex;
4497   dm->ops->load                      = DMLoad_Plex;
4498   dm->ops->setfromoptions            = DMSetFromOptions_Plex;
4499   dm->ops->clone                     = DMClone_Plex;
4500   dm->ops->setup                     = DMSetUp_Plex;
4501   dm->ops->createlocalsection        = DMCreateLocalSection_Plex;
4502   dm->ops->createdefaultconstraints  = DMCreateDefaultConstraints_Plex;
4503   dm->ops->createglobalvector        = DMCreateGlobalVector_Plex;
4504   dm->ops->createlocalvector         = DMCreateLocalVector_Plex;
4505   dm->ops->getlocaltoglobalmapping   = NULL;
4506   dm->ops->createfieldis             = NULL;
4507   dm->ops->createcoordinatedm        = DMCreateCoordinateDM_Plex;
4508   dm->ops->createcoordinatefield     = DMCreateCoordinateField_Plex;
4509   dm->ops->getcoloring               = NULL;
4510   dm->ops->creatematrix              = DMCreateMatrix_Plex;
4511   dm->ops->createinterpolation       = DMCreateInterpolation_Plex;
4512   dm->ops->createmassmatrix          = DMCreateMassMatrix_Plex;
4513   dm->ops->createmassmatrixlumped    = DMCreateMassMatrixLumped_Plex;
4514   dm->ops->createinjection           = DMCreateInjection_Plex;
4515   dm->ops->refine                    = DMRefine_Plex;
4516   dm->ops->coarsen                   = DMCoarsen_Plex;
4517   dm->ops->refinehierarchy           = DMRefineHierarchy_Plex;
4518   dm->ops->coarsenhierarchy          = DMCoarsenHierarchy_Plex;
4519   dm->ops->extrude                   = DMExtrude_Plex;
4520   dm->ops->globaltolocalbegin        = NULL;
4521   dm->ops->globaltolocalend          = NULL;
4522   dm->ops->localtoglobalbegin        = NULL;
4523   dm->ops->localtoglobalend          = NULL;
4524   dm->ops->destroy                   = DMDestroy_Plex;
4525   dm->ops->createsubdm               = DMCreateSubDM_Plex;
4526   dm->ops->createsuperdm             = DMCreateSuperDM_Plex;
4527   dm->ops->getdimpoints              = DMGetDimPoints_Plex;
4528   dm->ops->locatepoints              = DMLocatePoints_Plex;
4529   dm->ops->projectfunctionlocal      = DMProjectFunctionLocal_Plex;
4530   dm->ops->projectfunctionlabellocal = DMProjectFunctionLabelLocal_Plex;
4531   dm->ops->projectfieldlocal         = DMProjectFieldLocal_Plex;
4532   dm->ops->projectfieldlabellocal    = DMProjectFieldLabelLocal_Plex;
4533   dm->ops->projectbdfieldlabellocal  = DMProjectBdFieldLabelLocal_Plex;
4534   dm->ops->computel2diff             = DMComputeL2Diff_Plex;
4535   dm->ops->computel2gradientdiff     = DMComputeL2GradientDiff_Plex;
4536   dm->ops->computel2fielddiff        = DMComputeL2FieldDiff_Plex;
4537   dm->ops->getneighbors              = DMGetNeighbors_Plex;
4538   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertBoundaryValues_C", DMPlexInsertBoundaryValues_Plex));
4539   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertTimeDerviativeBoundaryValues_C", DMPlexInsertTimeDerivativeBoundaryValues_Plex));
4540   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMSetUpGLVisViewer_C", DMSetUpGLVisViewer_Plex));
4541   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMCreateNeumannOverlap_C", DMCreateNeumannOverlap_Plex));
4542   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4543   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeGetDefault_C", DMPlexDistributeGetDefault_Plex));
4544   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeSetDefault_C", DMPlexDistributeSetDefault_Plex));
4545   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderGetDefault_C", DMPlexReorderGetDefault_Plex));
4546   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderSetDefault_C", DMPlexReorderSetDefault_Plex));
4547   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMInterpolateSolution_C", DMInterpolateSolution_Plex));
4548   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
4549   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetOverlap_C", DMPlexSetOverlap_Plex));
4550   PetscFunctionReturn(PETSC_SUCCESS);
4551 }
4552 
4553 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
4554 {
4555   DM_Plex *mesh = (DM_Plex *)dm->data;
4556   PetscSF  face_sf;
4557 
4558   PetscFunctionBegin;
4559   mesh->refct++;
4560   (*newdm)->data = mesh;
4561   PetscCall(DMPlexGetIsoperiodicFaceSF(dm, &face_sf));
4562   PetscCall(DMPlexSetIsoperiodicFaceSF(*newdm, face_sf));
4563   PetscCall(PetscObjectChangeTypeName((PetscObject)*newdm, DMPLEX));
4564   PetscCall(DMInitialize_Plex(*newdm));
4565   PetscFunctionReturn(PETSC_SUCCESS);
4566 }
4567 
4568 /*MC
4569   DMPLEX = "plex" - A `DM` object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
4570                     In the local representation, `Vec`s contain all unknowns in the interior and shared boundary. This is
4571                     specified by a PetscSection object. Ownership in the global representation is determined by
4572                     ownership of the underlying `DMPLEX` points. This is specified by another `PetscSection` object.
4573 
4574   Options Database Keys:
4575 + -dm_refine_pre                     - Refine mesh before distribution
4576 + -dm_refine_uniform_pre             - Choose uniform or generator-based refinement
4577 + -dm_refine_volume_limit_pre        - Cell volume limit after pre-refinement using generator
4578 . -dm_distribute                     - Distribute mesh across processes
4579 . -dm_distribute_overlap             - Number of cells to overlap for distribution
4580 . -dm_refine                         - Refine mesh after distribution
4581 . -dm_plex_hash_location             - Use grid hashing for point location
4582 . -dm_plex_hash_box_faces <n,m,p>    - The number of divisions in each direction of the grid hash
4583 . -dm_plex_partition_balance         - Attempt to evenly divide points on partition boundary between processes
4584 . -dm_plex_remesh_bd                 - Allow changes to the boundary on remeshing
4585 . -dm_plex_max_projection_height     - Maximum mesh point height used to project locally
4586 . -dm_plex_regular_refinement        - Use special nested projection algorithm for regular refinement
4587 . -dm_plex_check_all                 - Perform all checks below
4588 . -dm_plex_check_symmetry            - Check that the adjacency information in the mesh is symmetric
4589 . -dm_plex_check_skeleton <celltype> - Check that each cell has the correct number of vertices
4590 . -dm_plex_check_faces <celltype>    - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
4591 . -dm_plex_check_geometry            - Check that cells have positive volume
4592 . -dm_view :mesh.tex:ascii_latex     - View the mesh in LaTeX/TikZ
4593 . -dm_plex_view_scale <num>          - Scale the TikZ
4594 - -dm_plex_print_fem <num>           - View FEM assembly information, such as element vectors and matrices
4595 
4596   Level: intermediate
4597 
4598 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`, `PetscSection`
4599 M*/
4600 
4601 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
4602 {
4603   DM_Plex *mesh;
4604   PetscInt unit;
4605 
4606   PetscFunctionBegin;
4607   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4608   PetscCall(PetscNew(&mesh));
4609   dm->data = mesh;
4610 
4611   mesh->refct = 1;
4612   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection));
4613   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection));
4614   mesh->refinementUniform      = PETSC_TRUE;
4615   mesh->refinementLimit        = -1.0;
4616   mesh->distDefault            = PETSC_TRUE;
4617   mesh->reorderDefault         = DMPLEX_REORDER_DEFAULT_NOTSET;
4618   mesh->distributionName       = NULL;
4619   mesh->interpolated           = DMPLEX_INTERPOLATED_INVALID;
4620   mesh->interpolatedCollective = DMPLEX_INTERPOLATED_INVALID;
4621 
4622   PetscCall(PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner));
4623   mesh->remeshBd = PETSC_FALSE;
4624 
4625   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
4626 
4627   mesh->depthState    = -1;
4628   mesh->celltypeState = -1;
4629   mesh->printTol      = 1.0e-10;
4630 
4631   PetscCall(DMInitialize_Plex(dm));
4632   PetscFunctionReturn(PETSC_SUCCESS);
4633 }
4634 
4635 /*@
4636   DMPlexCreate - Creates a `DMPLEX` object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
4637 
4638   Collective
4639 
4640   Input Parameter:
4641 . comm - The communicator for the `DMPLEX` object
4642 
4643   Output Parameter:
4644 . mesh - The `DMPLEX` object
4645 
4646   Level: beginner
4647 
4648 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMType`, `DMCreate()`, `DMSetType()`
4649 @*/
4650 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
4651 {
4652   PetscFunctionBegin;
4653   PetscAssertPointer(mesh, 2);
4654   PetscCall(DMCreate(comm, mesh));
4655   PetscCall(DMSetType(*mesh, DMPLEX));
4656   PetscFunctionReturn(PETSC_SUCCESS);
4657 }
4658 
4659 /*@C
4660   DMPlexBuildFromCellListParallel - Build distributed `DMPLEX` topology from a list of vertices for each cell (common mesh generator output)
4661 
4662   Collective; No Fortran Support
4663 
4664   Input Parameters:
4665 + dm          - The `DM`
4666 . numCells    - The number of cells owned by this process
4667 . numVertices - The number of vertices to be owned by this process, or `PETSC_DECIDE`
4668 . NVertices   - The global number of vertices, or `PETSC_DETERMINE`
4669 . numCorners  - The number of vertices for each cell
4670 - cells       - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4671 
4672   Output Parameters:
4673 + vertexSF         - (Optional) `PetscSF` describing complete vertex ownership
4674 - verticesAdjSaved - (Optional) vertex adjacency array
4675 
4676   Level: advanced
4677 
4678   Notes:
4679   Two triangles sharing a face
4680 .vb
4681 
4682         2
4683       / | \
4684      /  |  \
4685     /   |   \
4686    0  0 | 1  3
4687     \   |   /
4688      \  |  /
4689       \ | /
4690         1
4691 .ve
4692   would have input
4693 .vb
4694   numCells = 2, numVertices = 4
4695   cells = [0 1 2  1 3 2]
4696 .ve
4697   which would result in the `DMPLEX`
4698 .vb
4699 
4700         4
4701       / | \
4702      /  |  \
4703     /   |   \
4704    2  0 | 1  5
4705     \   |   /
4706      \  |  /
4707       \ | /
4708         3
4709 .ve
4710 
4711   Vertices are implicitly numbered consecutively 0,...,NVertices.
4712   Each rank owns a chunk of numVertices consecutive vertices.
4713   If numVertices is `PETSC_DECIDE`, PETSc will distribute them as evenly as possible using PetscLayout.
4714   If NVertices is `PETSC_DETERMINE` and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
4715   If only NVertices is `PETSC_DETERMINE`, it is computed as the sum of numVertices over all ranks.
4716 
4717   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
4718 
4719 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildCoordinatesFromCellListParallel()`,
4720           `PetscSF`
4721 @*/
4722 PetscErrorCode DMPlexBuildFromCellListParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, const PetscInt cells[], PetscSF *vertexSF, PetscInt **verticesAdjSaved)
4723 {
4724   PetscSF     sfPoint;
4725   PetscLayout layout;
4726   PetscInt    numVerticesAdj, *verticesAdj, *cones, c, p;
4727 
4728   PetscFunctionBegin;
4729   PetscValidLogicalCollectiveInt(dm, NVertices, 4);
4730   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4731   /* Get/check global number of vertices */
4732   {
4733     PetscInt       NVerticesInCells, i;
4734     const PetscInt len = numCells * numCorners;
4735 
4736     /* NVerticesInCells = max(cells) + 1 */
4737     NVerticesInCells = PETSC_MIN_INT;
4738     for (i = 0; i < len; i++)
4739       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
4740     ++NVerticesInCells;
4741     PetscCall(MPIU_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
4742 
4743     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
4744     else
4745       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
4746   }
4747   /* Count locally unique vertices */
4748   {
4749     PetscHSetI vhash;
4750     PetscInt   off = 0;
4751 
4752     PetscCall(PetscHSetICreate(&vhash));
4753     for (c = 0; c < numCells; ++c) {
4754       for (p = 0; p < numCorners; ++p) PetscCall(PetscHSetIAdd(vhash, cells[c * numCorners + p]));
4755     }
4756     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
4757     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
4758     else verticesAdj = *verticesAdjSaved;
4759     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
4760     PetscCall(PetscHSetIDestroy(&vhash));
4761     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
4762   }
4763   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
4764   /* Create cones */
4765   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
4766   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
4767   PetscCall(DMSetUp(dm));
4768   PetscCall(DMPlexGetCones(dm, &cones));
4769   for (c = 0; c < numCells; ++c) {
4770     for (p = 0; p < numCorners; ++p) {
4771       const PetscInt gv = cells[c * numCorners + p];
4772       PetscInt       lv;
4773 
4774       /* Positions within verticesAdj form 0-based local vertex numbering;
4775          we need to shift it by numCells to get correct DAG points (cells go first) */
4776       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
4777       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
4778       cones[c * numCorners + p] = lv + numCells;
4779     }
4780   }
4781   /* Build point sf */
4782   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
4783   PetscCall(PetscLayoutSetSize(layout, NVertices));
4784   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
4785   PetscCall(PetscLayoutSetBlockSize(layout, 1));
4786   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
4787   PetscCall(PetscLayoutDestroy(&layout));
4788   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
4789   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
4790   if (dm->sf) {
4791     const char *prefix;
4792 
4793     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
4794     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
4795   }
4796   PetscCall(DMSetPointSF(dm, sfPoint));
4797   PetscCall(PetscSFDestroy(&sfPoint));
4798   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)(*vertexSF), "Vertex Ownership SF"));
4799   /* Fill in the rest of the topology structure */
4800   PetscCall(DMPlexSymmetrize(dm));
4801   PetscCall(DMPlexStratify(dm));
4802   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4803   PetscFunctionReturn(PETSC_SUCCESS);
4804 }
4805 
4806 /*@C
4807   DMPlexBuildCoordinatesFromCellListParallel - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
4808 
4809   Collective; No Fortran Support
4810 
4811   Input Parameters:
4812 + dm           - The `DM`
4813 . spaceDim     - The spatial dimension used for coordinates
4814 . sfVert       - `PetscSF` describing complete vertex ownership
4815 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4816 
4817   Level: advanced
4818 
4819 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellListParallel()`
4820 @*/
4821 PetscErrorCode DMPlexBuildCoordinatesFromCellListParallel(DM dm, PetscInt spaceDim, PetscSF sfVert, const PetscReal vertexCoords[])
4822 {
4823   PetscSection coordSection;
4824   Vec          coordinates;
4825   PetscScalar *coords;
4826   PetscInt     numVertices, numVerticesAdj, coordSize, v, vStart, vEnd;
4827 
4828   PetscFunctionBegin;
4829   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4830   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
4831   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
4832   PetscCall(DMSetCoordinateDim(dm, spaceDim));
4833   PetscCall(PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL));
4834   PetscCheck(vEnd - vStart == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Supplied sfVert has wrong number of leaves = %" PetscInt_FMT " != %" PetscInt_FMT " = vEnd - vStart", numVerticesAdj, vEnd - vStart);
4835   PetscCall(DMGetCoordinateSection(dm, &coordSection));
4836   PetscCall(PetscSectionSetNumFields(coordSection, 1));
4837   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
4838   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
4839   for (v = vStart; v < vEnd; ++v) {
4840     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
4841     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
4842   }
4843   PetscCall(PetscSectionSetUp(coordSection));
4844   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
4845   PetscCall(VecCreate(PetscObjectComm((PetscObject)dm), &coordinates));
4846   PetscCall(VecSetBlockSize(coordinates, spaceDim));
4847   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
4848   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
4849   PetscCall(VecSetType(coordinates, VECSTANDARD));
4850   PetscCall(VecGetArray(coordinates, &coords));
4851   {
4852     MPI_Datatype coordtype;
4853 
4854     /* Need a temp buffer for coords if we have complex/single */
4855     PetscCallMPI(MPI_Type_contiguous(spaceDim, MPIU_SCALAR, &coordtype));
4856     PetscCallMPI(MPI_Type_commit(&coordtype));
4857 #if defined(PETSC_USE_COMPLEX)
4858     {
4859       PetscScalar *svertexCoords;
4860       PetscInt     i;
4861       PetscCall(PetscMalloc1(numVertices * spaceDim, &svertexCoords));
4862       for (i = 0; i < numVertices * spaceDim; i++) svertexCoords[i] = vertexCoords[i];
4863       PetscCall(PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4864       PetscCall(PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
4865       PetscCall(PetscFree(svertexCoords));
4866     }
4867 #else
4868     PetscCall(PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4869     PetscCall(PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
4870 #endif
4871     PetscCallMPI(MPI_Type_free(&coordtype));
4872   }
4873   PetscCall(VecRestoreArray(coordinates, &coords));
4874   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
4875   PetscCall(VecDestroy(&coordinates));
4876   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
4877   PetscFunctionReturn(PETSC_SUCCESS);
4878 }
4879 
4880 /*@
4881   DMPlexCreateFromCellListParallelPetsc - Create distributed `DMPLEX` from a list of vertices for each cell (common mesh generator output)
4882 
4883   Collective
4884 
4885   Input Parameters:
4886 + comm         - The communicator
4887 . dim          - The topological dimension of the mesh
4888 . numCells     - The number of cells owned by this process
4889 . numVertices  - The number of vertices owned by this process, or `PETSC_DECIDE`
4890 . NVertices    - The global number of vertices, or `PETSC_DECIDE`
4891 . numCorners   - The number of vertices for each cell
4892 . interpolate  - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
4893 . cells        - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4894 . spaceDim     - The spatial dimension used for coordinates
4895 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
4896 
4897   Output Parameters:
4898 + dm          - The `DM`
4899 . vertexSF    - (Optional) `PetscSF` describing complete vertex ownership
4900 - verticesAdj - (Optional) vertex adjacency array
4901 
4902   Level: intermediate
4903 
4904   Notes:
4905   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`,
4906   `DMPlexBuildFromCellListParallel()`, `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellListParallel()`
4907 
4908   See `DMPlexBuildFromCellListParallel()` for an example and details about the topology-related parameters.
4909 
4910   See `DMPlexBuildCoordinatesFromCellListParallel()` for details about the geometry-related parameters.
4911 
4912 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
4913 @*/
4914 PetscErrorCode DMPlexCreateFromCellListParallelPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, PetscInt **verticesAdj, DM *dm)
4915 {
4916   PetscSF sfVert;
4917 
4918   PetscFunctionBegin;
4919   PetscCall(DMCreate(comm, dm));
4920   PetscCall(DMSetType(*dm, DMPLEX));
4921   PetscValidLogicalCollectiveInt(*dm, dim, 2);
4922   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
4923   PetscCall(DMSetDimension(*dm, dim));
4924   PetscCall(DMPlexBuildFromCellListParallel(*dm, numCells, numVertices, NVertices, numCorners, cells, &sfVert, verticesAdj));
4925   if (interpolate) {
4926     DM idm;
4927 
4928     PetscCall(DMPlexInterpolate(*dm, &idm));
4929     PetscCall(DMDestroy(dm));
4930     *dm = idm;
4931   }
4932   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
4933   if (vertexSF) *vertexSF = sfVert;
4934   else PetscCall(PetscSFDestroy(&sfVert));
4935   PetscFunctionReturn(PETSC_SUCCESS);
4936 }
4937 
4938 /*@C
4939   DMPlexBuildFromCellList - Build `DMPLEX` topology from a list of vertices for each cell (common mesh generator output)
4940 
4941   Collective; No Fortran Support
4942 
4943   Input Parameters:
4944 + dm          - The `DM`
4945 . numCells    - The number of cells owned by this process
4946 . numVertices - The number of vertices owned by this process, or `PETSC_DETERMINE`
4947 . numCorners  - The number of vertices for each cell
4948 - cells       - An array of numCells*numCorners numbers, the global vertex numbers for each cell
4949 
4950   Level: advanced
4951 
4952   Notes:
4953   Two triangles sharing a face
4954 .vb
4955 
4956         2
4957       / | \
4958      /  |  \
4959     /   |   \
4960    0  0 | 1  3
4961     \   |   /
4962      \  |  /
4963       \ | /
4964         1
4965 .ve
4966   would have input
4967 .vb
4968   numCells = 2, numVertices = 4
4969   cells = [0 1 2  1 3 2]
4970 .ve
4971   which would result in the `DMPLEX`
4972 .vb
4973 
4974         4
4975       / | \
4976      /  |  \
4977     /   |   \
4978    2  0 | 1  5
4979     \   |   /
4980      \  |  /
4981       \ | /
4982         3
4983 .ve
4984 
4985   If numVertices is `PETSC_DETERMINE`, it is computed by PETSc as the maximum vertex index in cells + 1.
4986 
4987 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListPetsc()`
4988 @*/
4989 PetscErrorCode DMPlexBuildFromCellList(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const PetscInt cells[])
4990 {
4991   PetscInt *cones, c, p, dim;
4992 
4993   PetscFunctionBegin;
4994   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
4995   PetscCall(DMGetDimension(dm, &dim));
4996   /* Get/check global number of vertices */
4997   {
4998     PetscInt       NVerticesInCells, i;
4999     const PetscInt len = numCells * numCorners;
5000 
5001     /* NVerticesInCells = max(cells) + 1 */
5002     NVerticesInCells = PETSC_MIN_INT;
5003     for (i = 0; i < len; i++)
5004       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
5005     ++NVerticesInCells;
5006 
5007     if (numVertices == PETSC_DECIDE) numVertices = NVerticesInCells;
5008     else
5009       PetscCheck(numVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, numVertices, NVerticesInCells);
5010   }
5011   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
5012   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
5013   PetscCall(DMSetUp(dm));
5014   PetscCall(DMPlexGetCones(dm, &cones));
5015   for (c = 0; c < numCells; ++c) {
5016     for (p = 0; p < numCorners; ++p) cones[c * numCorners + p] = cells[c * numCorners + p] + numCells;
5017   }
5018   PetscCall(DMPlexSymmetrize(dm));
5019   PetscCall(DMPlexStratify(dm));
5020   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5021   PetscFunctionReturn(PETSC_SUCCESS);
5022 }
5023 
5024 /*@C
5025   DMPlexBuildCoordinatesFromCellList - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
5026 
5027   Collective; No Fortran Support
5028 
5029   Input Parameters:
5030 + dm           - The `DM`
5031 . spaceDim     - The spatial dimension used for coordinates
5032 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
5033 
5034   Level: advanced
5035 
5036 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellList()`
5037 @*/
5038 PetscErrorCode DMPlexBuildCoordinatesFromCellList(DM dm, PetscInt spaceDim, const PetscReal vertexCoords[])
5039 {
5040   PetscSection coordSection;
5041   Vec          coordinates;
5042   DM           cdm;
5043   PetscScalar *coords;
5044   PetscInt     v, vStart, vEnd, d;
5045 
5046   PetscFunctionBegin;
5047   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
5048   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
5049   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
5050   PetscCall(DMSetCoordinateDim(dm, spaceDim));
5051   PetscCall(DMGetCoordinateSection(dm, &coordSection));
5052   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5053   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
5054   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
5055   for (v = vStart; v < vEnd; ++v) {
5056     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
5057     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
5058   }
5059   PetscCall(PetscSectionSetUp(coordSection));
5060 
5061   PetscCall(DMGetCoordinateDM(dm, &cdm));
5062   PetscCall(DMCreateLocalVector(cdm, &coordinates));
5063   PetscCall(VecSetBlockSize(coordinates, spaceDim));
5064   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5065   PetscCall(VecGetArrayWrite(coordinates, &coords));
5066   for (v = 0; v < vEnd - vStart; ++v) {
5067     for (d = 0; d < spaceDim; ++d) coords[v * spaceDim + d] = vertexCoords[v * spaceDim + d];
5068   }
5069   PetscCall(VecRestoreArrayWrite(coordinates, &coords));
5070   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
5071   PetscCall(VecDestroy(&coordinates));
5072   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
5073   PetscFunctionReturn(PETSC_SUCCESS);
5074 }
5075 
5076 /*@
5077   DMPlexCreateFromCellListPetsc - Create `DMPLEX` from a list of vertices for each cell (common mesh generator output), but only process 0 takes in the input
5078 
5079   Collective
5080 
5081   Input Parameters:
5082 + comm         - The communicator
5083 . dim          - The topological dimension of the mesh
5084 . numCells     - The number of cells, only on process 0
5085 . numVertices  - The number of vertices owned by this process, or `PETSC_DECIDE`, only on process 0
5086 . numCorners   - The number of vertices for each cell, only on process 0
5087 . interpolate  - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
5088 . cells        - An array of numCells*numCorners numbers, the vertices for each cell, only on process 0
5089 . spaceDim     - The spatial dimension used for coordinates
5090 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex, only on process 0
5091 
5092   Output Parameter:
5093 . dm - The `DM`, which only has points on process 0
5094 
5095   Level: intermediate
5096 
5097   Notes:
5098   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`, `DMPlexBuildFromCellList()`,
5099   `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellList()`
5100 
5101   See `DMPlexBuildFromCellList()` for an example and details about the topology-related parameters.
5102   See `DMPlexBuildCoordinatesFromCellList()` for details about the geometry-related parameters.
5103   See `DMPlexCreateFromCellListParallelPetsc()` for parallel input
5104 
5105 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellList()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
5106 @*/
5107 PetscErrorCode DMPlexCreateFromCellListPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], DM *dm)
5108 {
5109   PetscMPIInt rank;
5110 
5111   PetscFunctionBegin;
5112   PetscCheck(dim, comm, PETSC_ERR_ARG_OUTOFRANGE, "This is not appropriate for 0-dimensional meshes. Consider either creating the DM using DMPlexCreateFromDAG(), by hand, or using DMSwarm.");
5113   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5114   PetscCall(DMCreate(comm, dm));
5115   PetscCall(DMSetType(*dm, DMPLEX));
5116   PetscCall(DMSetDimension(*dm, dim));
5117   if (rank == 0) PetscCall(DMPlexBuildFromCellList(*dm, numCells, numVertices, numCorners, cells));
5118   else PetscCall(DMPlexBuildFromCellList(*dm, 0, 0, 0, NULL));
5119   if (interpolate) {
5120     DM idm;
5121 
5122     PetscCall(DMPlexInterpolate(*dm, &idm));
5123     PetscCall(DMDestroy(dm));
5124     *dm = idm;
5125   }
5126   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, vertexCoords));
5127   else PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, NULL));
5128   PetscFunctionReturn(PETSC_SUCCESS);
5129 }
5130 
5131 /*@
5132   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a `DM`
5133 
5134   Input Parameters:
5135 + dm               - The empty `DM` object, usually from `DMCreate()` and `DMSetDimension()`
5136 . depth            - The depth of the DAG
5137 . numPoints        - Array of size depth + 1 containing the number of points at each `depth`
5138 . coneSize         - The cone size of each point
5139 . cones            - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
5140 . coneOrientations - The orientation of each cone point
5141 - vertexCoords     - An array of `numPoints`[0]*spacedim numbers representing the coordinates of each vertex, with spacedim the value set via `DMSetCoordinateDim()`
5142 
5143   Output Parameter:
5144 . dm - The `DM`
5145 
5146   Level: advanced
5147 
5148   Note:
5149   Two triangles sharing a face would have input
5150 .vb
5151   depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
5152   cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
5153  vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
5154 .ve
5155   which would result in the DMPlex
5156 .vb
5157         4
5158       / | \
5159      /  |  \
5160     /   |   \
5161    2  0 | 1  5
5162     \   |   /
5163      \  |  /
5164       \ | /
5165         3
5166 .ve
5167   Notice that all points are numbered consecutively, unlike `DMPlexCreateFromCellListPetsc()`
5168 
5169 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
5170 @*/
5171 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
5172 {
5173   Vec          coordinates;
5174   PetscSection coordSection;
5175   PetscScalar *coords;
5176   PetscInt     coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
5177 
5178   PetscFunctionBegin;
5179   PetscCall(DMGetDimension(dm, &dim));
5180   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
5181   PetscCheck(dimEmbed >= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Embedding dimension %" PetscInt_FMT " cannot be less than intrinsic dimension %" PetscInt_FMT, dimEmbed, dim);
5182   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
5183   PetscCall(DMPlexSetChart(dm, pStart, pEnd));
5184   for (p = pStart; p < pEnd; ++p) {
5185     PetscCall(DMPlexSetConeSize(dm, p, coneSize[p - pStart]));
5186     if (firstVertex < 0 && !coneSize[p - pStart]) firstVertex = p - pStart;
5187   }
5188   PetscCheck(firstVertex >= 0 || !numPoints[0], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Expected %" PetscInt_FMT " vertices but could not find any", numPoints[0]);
5189   PetscCall(DMSetUp(dm)); /* Allocate space for cones */
5190   for (p = pStart, off = 0; p < pEnd; off += coneSize[p - pStart], ++p) {
5191     PetscCall(DMPlexSetCone(dm, p, &cones[off]));
5192     PetscCall(DMPlexSetConeOrientation(dm, p, &coneOrientations[off]));
5193   }
5194   PetscCall(DMPlexSymmetrize(dm));
5195   PetscCall(DMPlexStratify(dm));
5196   /* Build coordinates */
5197   PetscCall(DMGetCoordinateSection(dm, &coordSection));
5198   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5199   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dimEmbed));
5200   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numPoints[0]));
5201   for (v = firstVertex; v < firstVertex + numPoints[0]; ++v) {
5202     PetscCall(PetscSectionSetDof(coordSection, v, dimEmbed));
5203     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed));
5204   }
5205   PetscCall(PetscSectionSetUp(coordSection));
5206   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
5207   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
5208   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5209   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
5210   PetscCall(VecSetBlockSize(coordinates, dimEmbed));
5211   PetscCall(VecSetType(coordinates, VECSTANDARD));
5212   if (vertexCoords) {
5213     PetscCall(VecGetArray(coordinates, &coords));
5214     for (v = 0; v < numPoints[0]; ++v) {
5215       PetscInt off;
5216 
5217       PetscCall(PetscSectionGetOffset(coordSection, v + firstVertex, &off));
5218       for (d = 0; d < dimEmbed; ++d) coords[off + d] = vertexCoords[v * dimEmbed + d];
5219     }
5220   }
5221   PetscCall(VecRestoreArray(coordinates, &coords));
5222   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
5223   PetscCall(VecDestroy(&coordinates));
5224   PetscFunctionReturn(PETSC_SUCCESS);
5225 }
5226 
5227 /*@C
5228   DMPlexCreateCellVertexFromFile - Create a `DMPLEX` mesh from a simple cell-vertex file.
5229 
5230   Collective
5231 
5232 + comm        - The MPI communicator
5233 . filename    - Name of the .dat file
5234 - interpolate - Create faces and edges in the mesh
5235 
5236   Output Parameter:
5237 . dm  - The `DM` object representing the mesh
5238 
5239   Level: beginner
5240 
5241   Note:
5242   The format is the simplest possible:
5243 .vb
5244   Ne
5245   v0 v1 ... vk
5246   Nv
5247   x y z marker
5248 .ve
5249 
5250   Developer Note:
5251   Should use a `PetscViewer` not a filename
5252 
5253 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromFile()`, `DMPlexCreateMedFromFile()`, `DMPlexCreateGmsh()`, `DMPlexCreate()`
5254 @*/
5255 static PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
5256 {
5257   DMLabel      marker;
5258   PetscViewer  viewer;
5259   Vec          coordinates;
5260   PetscSection coordSection;
5261   PetscScalar *coords;
5262   char         line[PETSC_MAX_PATH_LEN];
5263   PetscInt     dim = 3, cdim = 3, coordSize, v, c, d;
5264   PetscMPIInt  rank;
5265   int          snum, Nv, Nc, Ncn, Nl;
5266 
5267   PetscFunctionBegin;
5268   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5269   PetscCall(PetscViewerCreate(comm, &viewer));
5270   PetscCall(PetscViewerSetType(viewer, PETSCVIEWERASCII));
5271   PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
5272   PetscCall(PetscViewerFileSetName(viewer, filename));
5273   if (rank == 0) {
5274     PetscCall(PetscViewerRead(viewer, line, 4, NULL, PETSC_STRING));
5275     snum = sscanf(line, "%d %d %d %d", &Nc, &Nv, &Ncn, &Nl);
5276     PetscCheck(snum == 4, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5277   } else {
5278     Nc = Nv = Ncn = Nl = 0;
5279   }
5280   PetscCall(DMCreate(comm, dm));
5281   PetscCall(DMSetType(*dm, DMPLEX));
5282   PetscCall(DMPlexSetChart(*dm, 0, Nc + Nv));
5283   PetscCall(DMSetDimension(*dm, dim));
5284   PetscCall(DMSetCoordinateDim(*dm, cdim));
5285   /* Read topology */
5286   if (rank == 0) {
5287     char     format[PETSC_MAX_PATH_LEN];
5288     PetscInt cone[8];
5289     int      vbuf[8], v;
5290 
5291     for (c = 0; c < Ncn; ++c) {
5292       format[c * 3 + 0] = '%';
5293       format[c * 3 + 1] = 'd';
5294       format[c * 3 + 2] = ' ';
5295     }
5296     format[Ncn * 3 - 1] = '\0';
5297     for (c = 0; c < Nc; ++c) PetscCall(DMPlexSetConeSize(*dm, c, Ncn));
5298     PetscCall(DMSetUp(*dm));
5299     for (c = 0; c < Nc; ++c) {
5300       PetscCall(PetscViewerRead(viewer, line, Ncn, NULL, PETSC_STRING));
5301       switch (Ncn) {
5302       case 2:
5303         snum = sscanf(line, format, &vbuf[0], &vbuf[1]);
5304         break;
5305       case 3:
5306         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2]);
5307         break;
5308       case 4:
5309         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3]);
5310         break;
5311       case 6:
5312         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5]);
5313         break;
5314       case 8:
5315         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);
5316         break;
5317       default:
5318         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No cell shape with %d vertices", Ncn);
5319       }
5320       PetscCheck(snum == Ncn, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5321       for (v = 0; v < Ncn; ++v) cone[v] = vbuf[v] + Nc;
5322       /* Hexahedra are inverted */
5323       if (Ncn == 8) {
5324         PetscInt tmp = cone[1];
5325         cone[1]      = cone[3];
5326         cone[3]      = tmp;
5327       }
5328       PetscCall(DMPlexSetCone(*dm, c, cone));
5329     }
5330   }
5331   PetscCall(DMPlexSymmetrize(*dm));
5332   PetscCall(DMPlexStratify(*dm));
5333   /* Read coordinates */
5334   PetscCall(DMGetCoordinateSection(*dm, &coordSection));
5335   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5336   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
5337   PetscCall(PetscSectionSetChart(coordSection, Nc, Nc + Nv));
5338   for (v = Nc; v < Nc + Nv; ++v) {
5339     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
5340     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
5341   }
5342   PetscCall(PetscSectionSetUp(coordSection));
5343   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
5344   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
5345   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5346   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
5347   PetscCall(VecSetBlockSize(coordinates, cdim));
5348   PetscCall(VecSetType(coordinates, VECSTANDARD));
5349   PetscCall(VecGetArray(coordinates, &coords));
5350   if (rank == 0) {
5351     char   format[PETSC_MAX_PATH_LEN];
5352     double x[3];
5353     int    l, val[3];
5354 
5355     if (Nl) {
5356       for (l = 0; l < Nl; ++l) {
5357         format[l * 3 + 0] = '%';
5358         format[l * 3 + 1] = 'd';
5359         format[l * 3 + 2] = ' ';
5360       }
5361       format[Nl * 3 - 1] = '\0';
5362       PetscCall(DMCreateLabel(*dm, "marker"));
5363       PetscCall(DMGetLabel(*dm, "marker", &marker));
5364     }
5365     for (v = 0; v < Nv; ++v) {
5366       PetscCall(PetscViewerRead(viewer, line, 3 + Nl, NULL, PETSC_STRING));
5367       snum = sscanf(line, "%lg %lg %lg", &x[0], &x[1], &x[2]);
5368       PetscCheck(snum == 3, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5369       switch (Nl) {
5370       case 0:
5371         snum = 0;
5372         break;
5373       case 1:
5374         snum = sscanf(line, format, &val[0]);
5375         break;
5376       case 2:
5377         snum = sscanf(line, format, &val[0], &val[1]);
5378         break;
5379       case 3:
5380         snum = sscanf(line, format, &val[0], &val[1], &val[2]);
5381         break;
5382       default:
5383         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Request support for %d labels", Nl);
5384       }
5385       PetscCheck(snum == Nl, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
5386       for (d = 0; d < cdim; ++d) coords[v * cdim + d] = x[d];
5387       for (l = 0; l < Nl; ++l) PetscCall(DMLabelSetValue(marker, v + Nc, val[l]));
5388     }
5389   }
5390   PetscCall(VecRestoreArray(coordinates, &coords));
5391   PetscCall(DMSetCoordinatesLocal(*dm, coordinates));
5392   PetscCall(VecDestroy(&coordinates));
5393   PetscCall(PetscViewerDestroy(&viewer));
5394   if (interpolate) {
5395     DM      idm;
5396     DMLabel bdlabel;
5397 
5398     PetscCall(DMPlexInterpolate(*dm, &idm));
5399     PetscCall(DMDestroy(dm));
5400     *dm = idm;
5401 
5402     if (!Nl) {
5403       PetscCall(DMCreateLabel(*dm, "marker"));
5404       PetscCall(DMGetLabel(*dm, "marker", &bdlabel));
5405       PetscCall(DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel));
5406       PetscCall(DMPlexLabelComplete(*dm, bdlabel));
5407     }
5408   }
5409   PetscFunctionReturn(PETSC_SUCCESS);
5410 }
5411 
5412 /*@C
5413   DMPlexCreateFromFile - This takes a filename and produces a `DM`
5414 
5415   Collective
5416 
5417   Input Parameters:
5418 + comm        - The communicator
5419 . filename    - A file name
5420 . plexname    - The object name of the resulting `DM`, also used for intra-datafile lookup by some formats
5421 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
5422 
5423   Output Parameter:
5424 . dm - The `DM`
5425 
5426   Options Database Key:
5427 . -dm_plex_create_from_hdf5_xdmf - use the `PETSC_VIEWER_HDF5_XDMF` format for reading HDF5
5428 
5429   Use `-dm_plex_create_ prefix` to pass options to the internal `PetscViewer`, e.g.
5430 $ -dm_plex_create_viewer_hdf5_collective
5431 
5432   Level: beginner
5433 
5434   Notes:
5435   Using `PETSCVIEWERHDF5` type with `PETSC_VIEWER_HDF5_PETSC` format, one can save multiple `DMPLEX`
5436   meshes in a single HDF5 file. This in turn requires one to name the `DMPLEX` object with `PetscObjectSetName()`
5437   before saving it with `DMView()` and before loading it with `DMLoad()` for identification of the mesh object.
5438   The input parameter name is thus used to name the `DMPLEX` object when `DMPlexCreateFromFile()` internally
5439   calls `DMLoad()`. Currently, name is ignored for other viewer types and/or formats.
5440 
5441 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`, `PetscObjectSetName()`, `DMView()`, `DMLoad()`
5442 @*/
5443 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], const char plexname[], PetscBool interpolate, DM *dm)
5444 {
5445   const char  extGmsh[]      = ".msh";
5446   const char  extGmsh2[]     = ".msh2";
5447   const char  extGmsh4[]     = ".msh4";
5448   const char  extCGNS[]      = ".cgns";
5449   const char  extExodus[]    = ".exo";
5450   const char  extExodus_e[]  = ".e";
5451   const char  extGenesis[]   = ".gen";
5452   const char  extFluent[]    = ".cas";
5453   const char  extHDF5[]      = ".h5";
5454   const char  extXDMFHDF5[]  = ".xdmf.h5";
5455   const char  extMed[]       = ".med";
5456   const char  extPLY[]       = ".ply";
5457   const char  extEGADSLite[] = ".egadslite";
5458   const char  extEGADS[]     = ".egads";
5459   const char  extIGES[]      = ".igs";
5460   const char  extSTEP[]      = ".stp";
5461   const char  extCV[]        = ".dat";
5462   size_t      len;
5463   PetscBool   isGmsh, isGmsh2, isGmsh4, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isMed, isPLY, isEGADSLite, isEGADS, isIGES, isSTEP, isCV, isXDMFHDF5;
5464   PetscMPIInt rank;
5465 
5466   PetscFunctionBegin;
5467   PetscAssertPointer(filename, 2);
5468   if (plexname) PetscAssertPointer(plexname, 3);
5469   PetscAssertPointer(dm, 5);
5470   PetscCall(DMInitializePackage());
5471   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5472   PetscCallMPI(MPI_Comm_rank(comm, &rank));
5473   PetscCall(PetscStrlen(filename, &len));
5474   PetscCheck(len, comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
5475 
5476 #define CheckExtension(extension__, is_extension__) \
5477   do { \
5478     PetscAssert(sizeof(extension__), comm, PETSC_ERR_PLIB, "Zero-size extension: %s", extension__); \
5479     /* don't count the null-terminator at the end */ \
5480     const size_t ext_len = sizeof(extension__) - 1; \
5481     if (len < ext_len) { \
5482       is_extension__ = PETSC_FALSE; \
5483     } else { \
5484       PetscCall(PetscStrncmp(filename + len - ext_len, extension__, ext_len, &is_extension__)); \
5485     } \
5486   } while (0)
5487 
5488   CheckExtension(extGmsh, isGmsh);
5489   CheckExtension(extGmsh2, isGmsh2);
5490   CheckExtension(extGmsh4, isGmsh4);
5491   CheckExtension(extCGNS, isCGNS);
5492   CheckExtension(extExodus, isExodus);
5493   if (!isExodus) CheckExtension(extExodus_e, isExodus);
5494   CheckExtension(extGenesis, isGenesis);
5495   CheckExtension(extFluent, isFluent);
5496   CheckExtension(extHDF5, isHDF5);
5497   CheckExtension(extMed, isMed);
5498   CheckExtension(extPLY, isPLY);
5499   CheckExtension(extEGADSLite, isEGADSLite);
5500   CheckExtension(extEGADS, isEGADS);
5501   CheckExtension(extIGES, isIGES);
5502   CheckExtension(extSTEP, isSTEP);
5503   CheckExtension(extCV, isCV);
5504   CheckExtension(extXDMFHDF5, isXDMFHDF5);
5505 
5506 #undef CheckExtension
5507 
5508   if (isGmsh || isGmsh2 || isGmsh4) {
5509     PetscCall(DMPlexCreateGmshFromFile(comm, filename, interpolate, dm));
5510   } else if (isCGNS) {
5511     PetscCall(DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm));
5512   } else if (isExodus || isGenesis) {
5513     PetscCall(DMPlexCreateExodusFromFile(comm, filename, interpolate, dm));
5514   } else if (isFluent) {
5515     PetscCall(DMPlexCreateFluentFromFile(comm, filename, interpolate, dm));
5516   } else if (isHDF5) {
5517     PetscViewer viewer;
5518 
5519     /* PETSC_VIEWER_HDF5_XDMF is used if the filename ends with .xdmf.h5, or if -dm_plex_create_from_hdf5_xdmf option is present */
5520     PetscCall(PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &isXDMFHDF5, NULL));
5521     PetscCall(PetscViewerCreate(comm, &viewer));
5522     PetscCall(PetscViewerSetType(viewer, PETSCVIEWERHDF5));
5523     PetscCall(PetscViewerSetOptionsPrefix(viewer, "dm_plex_create_"));
5524     PetscCall(PetscViewerSetFromOptions(viewer));
5525     PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
5526     PetscCall(PetscViewerFileSetName(viewer, filename));
5527 
5528     PetscCall(DMCreate(comm, dm));
5529     PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5530     PetscCall(DMSetType(*dm, DMPLEX));
5531     if (isXDMFHDF5) PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF));
5532     PetscCall(DMLoad(*dm, viewer));
5533     if (isXDMFHDF5) PetscCall(PetscViewerPopFormat(viewer));
5534     PetscCall(PetscViewerDestroy(&viewer));
5535 
5536     if (interpolate) {
5537       DM idm;
5538 
5539       PetscCall(DMPlexInterpolate(*dm, &idm));
5540       PetscCall(DMDestroy(dm));
5541       *dm = idm;
5542     }
5543   } else if (isMed) {
5544     PetscCall(DMPlexCreateMedFromFile(comm, filename, interpolate, dm));
5545   } else if (isPLY) {
5546     PetscCall(DMPlexCreatePLYFromFile(comm, filename, interpolate, dm));
5547   } else if (isEGADSLite || isEGADS || isIGES || isSTEP) {
5548     if (isEGADSLite) PetscCall(DMPlexCreateEGADSLiteFromFile(comm, filename, dm));
5549     else PetscCall(DMPlexCreateEGADSFromFile(comm, filename, dm));
5550     if (!interpolate) {
5551       DM udm;
5552 
5553       PetscCall(DMPlexUninterpolate(*dm, &udm));
5554       PetscCall(DMDestroy(dm));
5555       *dm = udm;
5556     }
5557   } else if (isCV) {
5558     PetscCall(DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm));
5559   } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
5560   PetscCall(PetscStrlen(plexname, &len));
5561   if (len) PetscCall(PetscObjectSetName((PetscObject)(*dm), plexname));
5562   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromFile, 0, 0, 0, 0));
5563   PetscFunctionReturn(PETSC_SUCCESS);
5564 }
5565 
5566 /*@C
5567   DMPlexCreateEphemeral - This takes a `DMPlexTransform` and a base `DMPlex` and produces an ephemeral `DM`, meaning one that is created on the fly in response to queries.
5568 
5569   Input Parameters:
5570 + tr     - The `DMPlexTransform`
5571 - prefix - An options prefix, or NULL
5572 
5573   Output Parameter:
5574 . dm - The `DM`
5575 
5576   Level: beginner
5577 
5578   Notes:
5579   An emphemeral mesh is one that is not stored concretely, as in the default `DMPLEX` implementation, but rather is produced on the fly in response to queries, using information from the transform and the base mesh.
5580 
5581 .seealso: `DMPlexCreateFromFile`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
5582 @*/
5583 PetscErrorCode DMPlexCreateEphemeral(DMPlexTransform tr, const char prefix[], DM *dm)
5584 {
5585   DM           bdm, bcdm, cdm;
5586   Vec          coordinates, coordinatesNew;
5587   PetscSection cs;
5588   PetscInt     dim, cdim, Nl;
5589 
5590   PetscFunctionBegin;
5591   PetscCall(DMCreate(PetscObjectComm((PetscObject)tr), dm));
5592   PetscCall(DMSetType(*dm, DMPLEX));
5593   ((DM_Plex *)(*dm)->data)->interpolated = DMPLEX_INTERPOLATED_FULL;
5594   // Handle coordinates
5595   PetscCall(DMPlexTransformGetDM(tr, &bdm));
5596   PetscCall(DMGetCoordinateDim(bdm, &cdim));
5597   PetscCall(DMSetCoordinateDim(*dm, cdim));
5598   PetscCall(DMGetDimension(bdm, &dim));
5599   PetscCall(DMSetDimension(*dm, dim));
5600   PetscCall(DMGetCoordinateDM(bdm, &bcdm));
5601   PetscCall(DMGetCoordinateDM(*dm, &cdm));
5602   PetscCall(DMCopyDisc(bcdm, cdm));
5603   PetscCall(DMGetLocalSection(cdm, &cs));
5604   PetscCall(PetscSectionSetNumFields(cs, 1));
5605   PetscCall(PetscSectionSetFieldComponents(cs, 0, cdim));
5606   PetscCall(DMGetCoordinatesLocal(bdm, &coordinates));
5607   PetscCall(VecDuplicate(coordinates, &coordinatesNew));
5608   PetscCall(VecCopy(coordinates, coordinatesNew));
5609   PetscCall(DMSetCoordinatesLocal(*dm, coordinatesNew));
5610   PetscCall(VecDestroy(&coordinatesNew));
5611 
5612   PetscCall(PetscObjectReference((PetscObject)tr));
5613   PetscCall(DMPlexTransformDestroy(&((DM_Plex *)(*dm)->data)->tr));
5614   ((DM_Plex *)(*dm)->data)->tr = tr;
5615   PetscCall(DMPlexDistributeSetDefault(*dm, PETSC_FALSE));
5616   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)*dm, prefix));
5617   PetscCall(DMSetFromOptions(*dm));
5618 
5619   PetscCall(DMGetNumLabels(bdm, &Nl));
5620   for (PetscInt l = 0; l < Nl; ++l) {
5621     DMLabel     label, labelNew;
5622     const char *lname;
5623     PetscBool   isDepth, isCellType;
5624 
5625     PetscCall(DMGetLabelName(bdm, l, &lname));
5626     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
5627     if (isDepth) continue;
5628     PetscCall(PetscStrcmp(lname, "celltype", &isCellType));
5629     if (isCellType) continue;
5630     PetscCall(DMCreateLabel(*dm, lname));
5631     PetscCall(DMGetLabel(bdm, lname, &label));
5632     PetscCall(DMGetLabel(*dm, lname, &labelNew));
5633     PetscCall(DMLabelSetType(labelNew, DMLABELEPHEMERAL));
5634     PetscCall(DMLabelEphemeralSetLabel(labelNew, label));
5635     PetscCall(DMLabelEphemeralSetTransform(labelNew, tr));
5636     PetscCall(DMLabelSetUp(labelNew));
5637   }
5638   PetscFunctionReturn(PETSC_SUCCESS);
5639 }
5640