xref: /petsc/src/dm/impls/plex/plexcreate.c (revision d7d2d1d2babbf975e9544da80073779625dd96d9)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h> /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>
4 #include <petscsf.h>
5 #include <petscdmplextransform.h> /*I   "petscdmplextransform.h"   I*/
6 #include <petscdmlabelephemeral.h>
7 #include <petsc/private/kernels/blockmatmult.h>
8 #include <petsc/private/kernels/blockinvert.h>
9 
10 #ifdef PETSC_HAVE_UNISTD_H
11   #include <unistd.h>
12 #endif
13 #include <errno.h>
14 
15 PetscLogEvent DMPLEX_CreateFromFile, DMPLEX_CreateFromOptions, DMPLEX_BuildFromCellList, DMPLEX_BuildCoordinatesFromCellList;
16 
17 /* External function declarations here */
18 static PetscErrorCode DMInitialize_Plex(DM dm);
19 
20 /* This copies internal things in the Plex structure that we generally want when making a new, related Plex */
21 PetscErrorCode DMPlexCopy_Internal(DM dmin, PetscBool copyPeriodicity, PetscBool copyOverlap, DM dmout)
22 {
23   const PetscReal     *maxCell, *Lstart, *L;
24   VecType              vecType;
25   MatType              matType;
26   PetscBool            dist, useCeed;
27   DMReorderDefaultFlag reorder;
28 
29   PetscFunctionBegin;
30   if (dmin == dmout) PetscFunctionReturn(PETSC_SUCCESS);
31   PetscCall(DMGetVecType(dmin, &vecType));
32   PetscCall(DMSetVecType(dmout, vecType));
33   PetscCall(DMGetMatType(dmin, &matType));
34   PetscCall(DMSetMatType(dmout, matType));
35   if (copyPeriodicity) {
36     PetscCall(DMGetPeriodicity(dmin, &maxCell, &Lstart, &L));
37     PetscCall(DMSetPeriodicity(dmout, maxCell, Lstart, L));
38     PetscCall(DMLocalizeCoordinates(dmout));
39   }
40   PetscCall(DMPlexDistributeGetDefault(dmin, &dist));
41   PetscCall(DMPlexDistributeSetDefault(dmout, dist));
42   PetscCall(DMPlexReorderGetDefault(dmin, &reorder));
43   PetscCall(DMPlexReorderSetDefault(dmout, reorder));
44   PetscCall(DMPlexGetUseCeed(dmin, &useCeed));
45   PetscCall(DMPlexSetUseCeed(dmout, useCeed));
46   ((DM_Plex *)dmout->data)->useHashLocation = ((DM_Plex *)dmin->data)->useHashLocation;
47   ((DM_Plex *)dmout->data)->printSetValues  = ((DM_Plex *)dmin->data)->printSetValues;
48   ((DM_Plex *)dmout->data)->printFEM        = ((DM_Plex *)dmin->data)->printFEM;
49   ((DM_Plex *)dmout->data)->printFVM        = ((DM_Plex *)dmin->data)->printFVM;
50   ((DM_Plex *)dmout->data)->printL2         = ((DM_Plex *)dmin->data)->printL2;
51   ((DM_Plex *)dmout->data)->printLocate     = ((DM_Plex *)dmin->data)->printLocate;
52   ((DM_Plex *)dmout->data)->printProject    = ((DM_Plex *)dmin->data)->printProject;
53   ((DM_Plex *)dmout->data)->printTol        = ((DM_Plex *)dmin->data)->printTol;
54   if (copyOverlap) PetscCall(DMPlexSetOverlap_Plex(dmout, dmin, 0));
55   PetscFunctionReturn(PETSC_SUCCESS);
56 }
57 
58 /* Replace dm with the contents of ndm, and then destroy ndm
59    - Share the DM_Plex structure
60    - Share the coordinates
61    - Share the SF
62 */
63 PetscErrorCode DMPlexReplace_Internal(DM dm, DM *ndm)
64 {
65   PetscSF          sf;
66   DM               dmNew = *ndm, coordDM, coarseDM;
67   Vec              coords;
68   const PetscReal *maxCell, *Lstart, *L;
69   PetscInt         dim, cdim;
70   PetscBool        use_natural;
71 
72   PetscFunctionBegin;
73   if (dm == dmNew) {
74     PetscCall(DMDestroy(ndm));
75     PetscFunctionReturn(PETSC_SUCCESS);
76   }
77   dm->setupcalled = dmNew->setupcalled;
78   if (!dm->hdr.name) {
79     const char *name;
80 
81     PetscCall(PetscObjectGetName((PetscObject)*ndm, &name));
82     PetscCall(PetscObjectSetName((PetscObject)dm, name));
83   }
84   PetscCall(DMGetDimension(dmNew, &dim));
85   PetscCall(DMSetDimension(dm, dim));
86   PetscCall(DMGetCoordinateDim(dmNew, &cdim));
87   PetscCall(DMSetCoordinateDim(dm, cdim));
88   PetscCall(DMGetPointSF(dmNew, &sf));
89   PetscCall(DMSetPointSF(dm, sf));
90   PetscCall(DMGetCoordinateDM(dmNew, &coordDM));
91   PetscCall(DMGetCoordinatesLocal(dmNew, &coords));
92   PetscCall(DMSetCoordinateDM(dm, coordDM));
93   PetscCall(DMSetCoordinatesLocal(dm, coords));
94   PetscCall(DMGetCellCoordinateDM(dmNew, &coordDM));
95   PetscCall(DMGetCellCoordinatesLocal(dmNew, &coords));
96   PetscCall(DMSetCellCoordinateDM(dm, coordDM));
97   PetscCall(DMSetCellCoordinatesLocal(dm, coords));
98   /* Do not want to create the coordinate field if it does not already exist, so do not call DMGetCoordinateField() */
99   PetscCall(DMFieldDestroy(&dm->coordinates[0].field));
100   dm->coordinates[0].field            = dmNew->coordinates[0].field;
101   ((DM_Plex *)dmNew->data)->coordFunc = ((DM_Plex *)dm->data)->coordFunc;
102   PetscCall(DMGetPeriodicity(dmNew, &maxCell, &Lstart, &L));
103   PetscCall(DMSetPeriodicity(dm, maxCell, Lstart, L));
104   PetscCall(DMGetNaturalSF(dmNew, &sf));
105   PetscCall(DMSetNaturalSF(dm, sf));
106   PetscCall(DMGetUseNatural(dmNew, &use_natural));
107   PetscCall(DMSetUseNatural(dm, use_natural));
108   PetscCall(DMDestroy_Plex(dm));
109   PetscCall(DMInitialize_Plex(dm));
110   dm->data = dmNew->data;
111   ((DM_Plex *)dmNew->data)->refct++;
112   {
113     PetscInt       num_face_sfs;
114     const PetscSF *sfs;
115     PetscCall(DMPlexGetIsoperiodicFaceSF(dm, &num_face_sfs, &sfs));
116     PetscCall(DMPlexSetIsoperiodicFaceSF(dm, num_face_sfs, (PetscSF *)sfs)); // for the compose function effect on dm
117   }
118   PetscCall(DMDestroyLabelLinkList_Internal(dm));
119   PetscCall(DMCopyLabels(dmNew, dm, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL));
120   PetscCall(DMGetCoarseDM(dmNew, &coarseDM));
121   PetscCall(DMSetCoarseDM(dm, coarseDM));
122   PetscCall(DMDestroy(ndm));
123   PetscFunctionReturn(PETSC_SUCCESS);
124 }
125 
126 /* Swap dm with the contents of dmNew
127    - Swap the DM_Plex structure
128    - Swap the coordinates
129    - Swap the point PetscSF
130 */
131 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
132 {
133   DM          coordDMA, coordDMB;
134   Vec         coordsA, coordsB;
135   PetscSF     sfA, sfB;
136   DMField     fieldTmp;
137   void       *tmp;
138   DMLabelLink listTmp;
139   DMLabel     depthTmp;
140   PetscInt    tmpI;
141 
142   PetscFunctionBegin;
143   if (dmA == dmB) PetscFunctionReturn(PETSC_SUCCESS);
144   PetscCall(DMGetPointSF(dmA, &sfA));
145   PetscCall(DMGetPointSF(dmB, &sfB));
146   PetscCall(PetscObjectReference((PetscObject)sfA));
147   PetscCall(DMSetPointSF(dmA, sfB));
148   PetscCall(DMSetPointSF(dmB, sfA));
149   PetscCall(PetscObjectDereference((PetscObject)sfA));
150 
151   PetscCall(DMGetCoordinateDM(dmA, &coordDMA));
152   PetscCall(DMGetCoordinateDM(dmB, &coordDMB));
153   PetscCall(PetscObjectReference((PetscObject)coordDMA));
154   PetscCall(DMSetCoordinateDM(dmA, coordDMB));
155   PetscCall(DMSetCoordinateDM(dmB, coordDMA));
156   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
157 
158   PetscCall(DMGetCoordinatesLocal(dmA, &coordsA));
159   PetscCall(DMGetCoordinatesLocal(dmB, &coordsB));
160   PetscCall(PetscObjectReference((PetscObject)coordsA));
161   PetscCall(DMSetCoordinatesLocal(dmA, coordsB));
162   PetscCall(DMSetCoordinatesLocal(dmB, coordsA));
163   PetscCall(PetscObjectDereference((PetscObject)coordsA));
164 
165   PetscCall(DMGetCellCoordinateDM(dmA, &coordDMA));
166   PetscCall(DMGetCellCoordinateDM(dmB, &coordDMB));
167   PetscCall(PetscObjectReference((PetscObject)coordDMA));
168   PetscCall(DMSetCellCoordinateDM(dmA, coordDMB));
169   PetscCall(DMSetCellCoordinateDM(dmB, coordDMA));
170   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
171 
172   PetscCall(DMGetCellCoordinatesLocal(dmA, &coordsA));
173   PetscCall(DMGetCellCoordinatesLocal(dmB, &coordsB));
174   PetscCall(PetscObjectReference((PetscObject)coordsA));
175   PetscCall(DMSetCellCoordinatesLocal(dmA, coordsB));
176   PetscCall(DMSetCellCoordinatesLocal(dmB, coordsA));
177   PetscCall(PetscObjectDereference((PetscObject)coordsA));
178 
179   fieldTmp                  = dmA->coordinates[0].field;
180   dmA->coordinates[0].field = dmB->coordinates[0].field;
181   dmB->coordinates[0].field = fieldTmp;
182   fieldTmp                  = dmA->coordinates[1].field;
183   dmA->coordinates[1].field = dmB->coordinates[1].field;
184   dmB->coordinates[1].field = fieldTmp;
185   tmp                       = dmA->data;
186   dmA->data                 = dmB->data;
187   dmB->data                 = tmp;
188   listTmp                   = dmA->labels;
189   dmA->labels               = dmB->labels;
190   dmB->labels               = listTmp;
191   depthTmp                  = dmA->depthLabel;
192   dmA->depthLabel           = dmB->depthLabel;
193   dmB->depthLabel           = depthTmp;
194   depthTmp                  = dmA->celltypeLabel;
195   dmA->celltypeLabel        = dmB->celltypeLabel;
196   dmB->celltypeLabel        = depthTmp;
197   tmpI                      = dmA->levelup;
198   dmA->levelup              = dmB->levelup;
199   dmB->levelup              = tmpI;
200   PetscFunctionReturn(PETSC_SUCCESS);
201 }
202 
203 PetscErrorCode DMPlexInterpolateInPlace_Internal(DM dm)
204 {
205   DM idm;
206 
207   PetscFunctionBegin;
208   PetscCall(DMPlexInterpolate(dm, &idm));
209   PetscCall(DMPlexCopyCoordinates(dm, idm));
210   PetscCall(DMPlexReplace_Internal(dm, &idm));
211   PetscFunctionReturn(PETSC_SUCCESS);
212 }
213 
214 /*@C
215   DMPlexCreateCoordinateSpace - Creates a finite element space for the coordinates
216 
217   Collective
218 
219   Input Parameters:
220 + dm        - The `DMPLEX`
221 . degree    - The degree of the finite element or `PETSC_DECIDE`
222 . project   - Flag to project current coordinates into the space
223 - coordFunc - An optional function to map new points from refinement to the surface
224 
225   Level: advanced
226 
227 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `PetscPointFunc`, `PetscFECreateLagrange()`, `DMGetCoordinateDM()`
228 @*/
229 PetscErrorCode DMPlexCreateCoordinateSpace(DM dm, PetscInt degree, PetscBool project, PetscPointFunc coordFunc)
230 {
231   DM_Plex *mesh = (DM_Plex *)dm->data;
232   PetscFE  fe   = NULL;
233   DM       cdm;
234   PetscInt dim, dE, qorder, height;
235 
236   PetscFunctionBegin;
237   PetscCall(DMGetDimension(dm, &dim));
238   PetscCall(DMGetCoordinateDim(dm, &dE));
239   qorder = degree;
240   PetscCall(DMGetCoordinateDM(dm, &cdm));
241   PetscObjectOptionsBegin((PetscObject)cdm);
242   PetscCall(PetscOptionsBoundedInt("-default_quadrature_order", "Quadrature order is one less than quadrature points per edge", "DMPlexCreateCoordinateSpace", qorder, &qorder, NULL, 0));
243   PetscOptionsEnd();
244   PetscCall(DMPlexGetVTKCellHeight(dm, &height));
245   if (degree >= 0) {
246     DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
247     PetscInt       cStart, cEnd, gct;
248 
249     PetscCall(DMPlexGetHeightStratum(dm, height, &cStart, &cEnd));
250     if (cEnd > cStart) PetscCall(DMPlexGetCellType(dm, cStart, &ct));
251     gct = (PetscInt)ct;
252     PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &gct, 1, MPIU_INT, MPI_MIN, PetscObjectComm((PetscObject)dm)));
253     ct = (DMPolytopeType)gct;
254     // Work around current bug in PetscDualSpaceSetUp_Lagrange()
255     //   Can be seen in plex_tutorials-ex10_1
256     if (ct != DM_POLYTOPE_SEG_PRISM_TENSOR && ct != DM_POLYTOPE_TRI_PRISM_TENSOR && ct != DM_POLYTOPE_QUAD_PRISM_TENSOR) PetscCall(PetscFECreateLagrangeByCell(PETSC_COMM_SELF, dim, dE, ct, degree, qorder, &fe));
257   }
258   PetscCall(DMSetCoordinateDisc(dm, fe, project));
259   PetscCall(PetscFEDestroy(&fe));
260   mesh->coordFunc = coordFunc;
261   PetscFunctionReturn(PETSC_SUCCESS);
262 }
263 
264 /*@
265   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
266 
267   Collective
268 
269   Input Parameters:
270 + comm            - The communicator for the `DM` object
271 . dim             - The spatial dimension
272 . simplex         - Flag for simplicial cells, otherwise they are tensor product cells
273 . interpolate     - Flag to create intermediate mesh pieces (edges, faces)
274 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
275 
276   Output Parameter:
277 . newdm - The `DM` object
278 
279   Level: beginner
280 
281 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetType()`, `DMCreate()`
282 @*/
283 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscReal refinementLimit, DM *newdm)
284 {
285   DM          dm;
286   PetscMPIInt rank;
287 
288   PetscFunctionBegin;
289   PetscCall(DMCreate(comm, &dm));
290   PetscCall(DMSetType(dm, DMPLEX));
291   PetscCall(DMSetDimension(dm, dim));
292   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
293   PetscCallMPI(MPI_Comm_rank(comm, &rank));
294   switch (dim) {
295   case 2:
296     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "triangular"));
297     else PetscCall(PetscObjectSetName((PetscObject)dm, "quadrilateral"));
298     break;
299   case 3:
300     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "tetrahedral"));
301     else PetscCall(PetscObjectSetName((PetscObject)dm, "hexahedral"));
302     break;
303   default:
304     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
305   }
306   if (rank) {
307     PetscInt numPoints[2] = {0, 0};
308     PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL));
309   } else {
310     switch (dim) {
311     case 2:
312       if (simplex) {
313         PetscInt    numPoints[2]        = {4, 2};
314         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
315         PetscInt    cones[6]            = {2, 3, 4, 5, 4, 3};
316         PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
317         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
318 
319         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
320       } else {
321         PetscInt    numPoints[2]        = {6, 2};
322         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
323         PetscInt    cones[8]            = {2, 3, 4, 5, 3, 6, 7, 4};
324         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
325         PetscScalar vertexCoords[12]    = {-1.0, -0.5, 0.0, -0.5, 0.0, 0.5, -1.0, 0.5, 1.0, -0.5, 1.0, 0.5};
326 
327         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
328       }
329       break;
330     case 3:
331       if (simplex) {
332         PetscInt    numPoints[2]        = {5, 2};
333         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
334         PetscInt    cones[8]            = {4, 3, 5, 2, 5, 3, 4, 6};
335         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
336         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0};
337 
338         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
339       } else {
340         PetscInt    numPoints[2]         = {12, 2};
341         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
342         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9, 5, 4, 10, 11, 7, 12, 13, 8};
343         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
344         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5, -1.0, 0.5, -0.5, 0.0, 0.5, -0.5, 0.0, -0.5, -0.5, -1.0, -0.5, 0.5, 0.0, -0.5, 0.5, 0.0, 0.5, 0.5, -1.0, 0.5, 0.5, 1.0, 0.5, -0.5, 1.0, -0.5, -0.5, 1.0, -0.5, 0.5, 1.0, 0.5, 0.5};
345 
346         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
347       }
348       break;
349     default:
350       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
351     }
352   }
353   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
354   *newdm = dm;
355   if (refinementLimit > 0.0) {
356     DM          rdm;
357     const char *name;
358 
359     PetscCall(DMPlexSetRefinementUniform(*newdm, PETSC_FALSE));
360     PetscCall(DMPlexSetRefinementLimit(*newdm, refinementLimit));
361     PetscCall(DMRefine(*newdm, comm, &rdm));
362     PetscCall(PetscObjectGetName((PetscObject)*newdm, &name));
363     PetscCall(PetscObjectSetName((PetscObject)rdm, name));
364     PetscCall(DMDestroy(newdm));
365     *newdm = rdm;
366   }
367   if (interpolate) {
368     DM idm;
369 
370     PetscCall(DMPlexInterpolate(*newdm, &idm));
371     PetscCall(DMDestroy(newdm));
372     *newdm = idm;
373   }
374   PetscFunctionReturn(PETSC_SUCCESS);
375 }
376 
377 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
378 {
379   const PetscInt numVertices    = 2;
380   PetscInt       markerRight    = 1;
381   PetscInt       markerLeft     = 1;
382   PetscBool      markerSeparate = PETSC_FALSE;
383   Vec            coordinates;
384   PetscSection   coordSection;
385   PetscScalar   *coords;
386   PetscInt       coordSize;
387   PetscMPIInt    rank;
388   PetscInt       cdim = 1, v;
389 
390   PetscFunctionBegin;
391   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
392   if (markerSeparate) {
393     markerRight = 2;
394     markerLeft  = 1;
395   }
396   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
397   if (rank == 0) {
398     PetscCall(DMPlexSetChart(dm, 0, numVertices));
399     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
400     PetscCall(DMSetLabelValue(dm, "marker", 0, markerLeft));
401     PetscCall(DMSetLabelValue(dm, "marker", 1, markerRight));
402   }
403   PetscCall(DMPlexSymmetrize(dm));
404   PetscCall(DMPlexStratify(dm));
405   /* Build coordinates */
406   PetscCall(DMSetCoordinateDim(dm, cdim));
407   PetscCall(DMGetCoordinateSection(dm, &coordSection));
408   PetscCall(PetscSectionSetNumFields(coordSection, 1));
409   PetscCall(PetscSectionSetChart(coordSection, 0, numVertices));
410   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
411   for (v = 0; v < numVertices; ++v) {
412     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
413     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
414   }
415   PetscCall(PetscSectionSetUp(coordSection));
416   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
417   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
418   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
419   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
420   PetscCall(VecSetBlockSize(coordinates, cdim));
421   PetscCall(VecSetType(coordinates, VECSTANDARD));
422   PetscCall(VecGetArray(coordinates, &coords));
423   coords[0] = lower[0];
424   coords[1] = upper[0];
425   PetscCall(VecRestoreArray(coordinates, &coords));
426   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
427   PetscCall(VecDestroy(&coordinates));
428   PetscFunctionReturn(PETSC_SUCCESS);
429 }
430 
431 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
432 {
433   const PetscInt numVertices    = (edges[0] + 1) * (edges[1] + 1);
434   const PetscInt numEdges       = edges[0] * (edges[1] + 1) + (edges[0] + 1) * edges[1];
435   PetscInt       markerTop      = 1;
436   PetscInt       markerBottom   = 1;
437   PetscInt       markerRight    = 1;
438   PetscInt       markerLeft     = 1;
439   PetscBool      markerSeparate = PETSC_FALSE;
440   Vec            coordinates;
441   PetscSection   coordSection;
442   PetscScalar   *coords;
443   PetscInt       coordSize;
444   PetscMPIInt    rank;
445   PetscInt       v, vx, vy;
446 
447   PetscFunctionBegin;
448   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
449   if (markerSeparate) {
450     markerTop    = 3;
451     markerBottom = 1;
452     markerRight  = 2;
453     markerLeft   = 4;
454   }
455   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
456   if (rank == 0) {
457     PetscInt e, ex, ey;
458 
459     PetscCall(DMPlexSetChart(dm, 0, numEdges + numVertices));
460     for (e = 0; e < numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
461     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
462     for (vx = 0; vx <= edges[0]; vx++) {
463       for (ey = 0; ey < edges[1]; ey++) {
464         PetscInt edge   = vx * edges[1] + ey + edges[0] * (edges[1] + 1);
465         PetscInt vertex = ey * (edges[0] + 1) + vx + numEdges;
466         PetscInt cone[2];
467 
468         cone[0] = vertex;
469         cone[1] = vertex + edges[0] + 1;
470         PetscCall(DMPlexSetCone(dm, edge, cone));
471         if (vx == edges[0]) {
472           PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
473           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
474           if (ey == edges[1] - 1) {
475             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
476             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerRight));
477           }
478         } else if (vx == 0) {
479           PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
480           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
481           if (ey == edges[1] - 1) {
482             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
483             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft));
484           }
485         }
486       }
487     }
488     for (vy = 0; vy <= edges[1]; vy++) {
489       for (ex = 0; ex < edges[0]; ex++) {
490         PetscInt edge   = vy * edges[0] + ex;
491         PetscInt vertex = vy * (edges[0] + 1) + ex + numEdges;
492         PetscInt cone[2];
493 
494         cone[0] = vertex;
495         cone[1] = vertex + 1;
496         PetscCall(DMPlexSetCone(dm, edge, cone));
497         if (vy == edges[1]) {
498           PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
499           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
500           if (ex == edges[0] - 1) {
501             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
502             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerTop));
503           }
504         } else if (vy == 0) {
505           PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
506           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
507           if (ex == edges[0] - 1) {
508             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
509             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom));
510           }
511         }
512       }
513     }
514   }
515   PetscCall(DMPlexSymmetrize(dm));
516   PetscCall(DMPlexStratify(dm));
517   /* Build coordinates */
518   PetscCall(DMSetCoordinateDim(dm, 2));
519   PetscCall(DMGetCoordinateSection(dm, &coordSection));
520   PetscCall(PetscSectionSetNumFields(coordSection, 1));
521   PetscCall(PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices));
522   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 2));
523   for (v = numEdges; v < numEdges + numVertices; ++v) {
524     PetscCall(PetscSectionSetDof(coordSection, v, 2));
525     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 2));
526   }
527   PetscCall(PetscSectionSetUp(coordSection));
528   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
529   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
530   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
531   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
532   PetscCall(VecSetBlockSize(coordinates, 2));
533   PetscCall(VecSetType(coordinates, VECSTANDARD));
534   PetscCall(VecGetArray(coordinates, &coords));
535   for (vy = 0; vy <= edges[1]; ++vy) {
536     for (vx = 0; vx <= edges[0]; ++vx) {
537       coords[(vy * (edges[0] + 1) + vx) * 2 + 0] = lower[0] + ((upper[0] - lower[0]) / edges[0]) * vx;
538       coords[(vy * (edges[0] + 1) + vx) * 2 + 1] = lower[1] + ((upper[1] - lower[1]) / edges[1]) * vy;
539     }
540   }
541   PetscCall(VecRestoreArray(coordinates, &coords));
542   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
543   PetscCall(VecDestroy(&coordinates));
544   PetscFunctionReturn(PETSC_SUCCESS);
545 }
546 
547 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
548 {
549   PetscInt     vertices[3], numVertices;
550   PetscInt     numFaces       = 2 * faces[0] * faces[1] + 2 * faces[1] * faces[2] + 2 * faces[0] * faces[2];
551   PetscInt     markerTop      = 1;
552   PetscInt     markerBottom   = 1;
553   PetscInt     markerFront    = 1;
554   PetscInt     markerBack     = 1;
555   PetscInt     markerRight    = 1;
556   PetscInt     markerLeft     = 1;
557   PetscBool    markerSeparate = PETSC_FALSE;
558   Vec          coordinates;
559   PetscSection coordSection;
560   PetscScalar *coords;
561   PetscInt     coordSize;
562   PetscMPIInt  rank;
563   PetscInt     v, vx, vy, vz;
564   PetscInt     voffset, iface = 0, cone[4];
565 
566   PetscFunctionBegin;
567   PetscCheck(faces[0] >= 1 && faces[1] >= 1 && faces[2] >= 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
568   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
569   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
570   if (markerSeparate) {
571     markerBottom = 1;
572     markerTop    = 2;
573     markerFront  = 3;
574     markerBack   = 4;
575     markerRight  = 5;
576     markerLeft   = 6;
577   }
578   vertices[0] = faces[0] + 1;
579   vertices[1] = faces[1] + 1;
580   vertices[2] = faces[2] + 1;
581   numVertices = vertices[0] * vertices[1] * vertices[2];
582   if (rank == 0) {
583     PetscInt f;
584 
585     PetscCall(DMPlexSetChart(dm, 0, numFaces + numVertices));
586     for (f = 0; f < numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
587     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
588 
589     /* Side 0 (Top) */
590     for (vy = 0; vy < faces[1]; vy++) {
591       for (vx = 0; vx < faces[0]; vx++) {
592         voffset = numFaces + vertices[0] * vertices[1] * (vertices[2] - 1) + vy * vertices[0] + vx;
593         cone[0] = voffset;
594         cone[1] = voffset + 1;
595         cone[2] = voffset + vertices[0] + 1;
596         cone[3] = voffset + vertices[0];
597         PetscCall(DMPlexSetCone(dm, iface, cone));
598         PetscCall(DMSetLabelValue(dm, "marker", iface, markerTop));
599         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerTop));
600         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerTop));
601         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerTop));
602         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerTop));
603         iface++;
604       }
605     }
606 
607     /* Side 1 (Bottom) */
608     for (vy = 0; vy < faces[1]; vy++) {
609       for (vx = 0; vx < faces[0]; vx++) {
610         voffset = numFaces + vy * (faces[0] + 1) + vx;
611         cone[0] = voffset + 1;
612         cone[1] = voffset;
613         cone[2] = voffset + vertices[0];
614         cone[3] = voffset + vertices[0] + 1;
615         PetscCall(DMPlexSetCone(dm, iface, cone));
616         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBottom));
617         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBottom));
618         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBottom));
619         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerBottom));
620         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerBottom));
621         iface++;
622       }
623     }
624 
625     /* Side 2 (Front) */
626     for (vz = 0; vz < faces[2]; vz++) {
627       for (vx = 0; vx < faces[0]; vx++) {
628         voffset = numFaces + vz * vertices[0] * vertices[1] + vx;
629         cone[0] = voffset;
630         cone[1] = voffset + 1;
631         cone[2] = voffset + vertices[0] * vertices[1] + 1;
632         cone[3] = voffset + vertices[0] * vertices[1];
633         PetscCall(DMPlexSetCone(dm, iface, cone));
634         PetscCall(DMSetLabelValue(dm, "marker", iface, markerFront));
635         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerFront));
636         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerFront));
637         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerFront));
638         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerFront));
639         iface++;
640       }
641     }
642 
643     /* Side 3 (Back) */
644     for (vz = 0; vz < faces[2]; vz++) {
645       for (vx = 0; vx < faces[0]; vx++) {
646         voffset = numFaces + vz * vertices[0] * vertices[1] + vertices[0] * (vertices[1] - 1) + vx;
647         cone[0] = voffset + vertices[0] * vertices[1];
648         cone[1] = voffset + vertices[0] * vertices[1] + 1;
649         cone[2] = voffset + 1;
650         cone[3] = voffset;
651         PetscCall(DMPlexSetCone(dm, iface, cone));
652         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBack));
653         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBack));
654         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBack));
655         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerBack));
656         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerBack));
657         iface++;
658       }
659     }
660 
661     /* Side 4 (Left) */
662     for (vz = 0; vz < faces[2]; vz++) {
663       for (vy = 0; vy < faces[1]; vy++) {
664         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0];
665         cone[0] = voffset;
666         cone[1] = voffset + vertices[0] * vertices[1];
667         cone[2] = voffset + vertices[0] * vertices[1] + vertices[0];
668         cone[3] = voffset + vertices[0];
669         PetscCall(DMPlexSetCone(dm, iface, cone));
670         PetscCall(DMSetLabelValue(dm, "marker", iface, markerLeft));
671         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerLeft));
672         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerLeft));
673         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[1] + 0, markerLeft));
674         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerLeft));
675         iface++;
676       }
677     }
678 
679     /* Side 5 (Right) */
680     for (vz = 0; vz < faces[2]; vz++) {
681       for (vy = 0; vy < faces[1]; vy++) {
682         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0] + faces[0];
683         cone[0] = voffset + vertices[0] * vertices[1];
684         cone[1] = voffset;
685         cone[2] = voffset + vertices[0];
686         cone[3] = voffset + vertices[0] * vertices[1] + vertices[0];
687         PetscCall(DMPlexSetCone(dm, iface, cone));
688         PetscCall(DMSetLabelValue(dm, "marker", iface, markerRight));
689         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerRight));
690         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerRight));
691         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerRight));
692         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerRight));
693         iface++;
694       }
695     }
696   }
697   PetscCall(DMPlexSymmetrize(dm));
698   PetscCall(DMPlexStratify(dm));
699   /* Build coordinates */
700   PetscCall(DMSetCoordinateDim(dm, 3));
701   PetscCall(DMGetCoordinateSection(dm, &coordSection));
702   PetscCall(PetscSectionSetNumFields(coordSection, 1));
703   PetscCall(PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices));
704   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 3));
705   for (v = numFaces; v < numFaces + numVertices; ++v) {
706     PetscCall(PetscSectionSetDof(coordSection, v, 3));
707     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 3));
708   }
709   PetscCall(PetscSectionSetUp(coordSection));
710   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
711   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
712   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
713   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
714   PetscCall(VecSetBlockSize(coordinates, 3));
715   PetscCall(VecSetType(coordinates, VECSTANDARD));
716   PetscCall(VecGetArray(coordinates, &coords));
717   for (vz = 0; vz <= faces[2]; ++vz) {
718     for (vy = 0; vy <= faces[1]; ++vy) {
719       for (vx = 0; vx <= faces[0]; ++vx) {
720         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 0] = lower[0] + ((upper[0] - lower[0]) / faces[0]) * vx;
721         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 1] = lower[1] + ((upper[1] - lower[1]) / faces[1]) * vy;
722         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 2] = lower[2] + ((upper[2] - lower[2]) / faces[2]) * vz;
723       }
724     }
725   }
726   PetscCall(VecRestoreArray(coordinates, &coords));
727   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
728   PetscCall(VecDestroy(&coordinates));
729   PetscFunctionReturn(PETSC_SUCCESS);
730 }
731 
732 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate)
733 {
734   PetscFunctionBegin;
735   PetscValidLogicalCollectiveInt(dm, dim, 2);
736   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
737   PetscCall(DMSetDimension(dm, dim - 1));
738   PetscCall(DMSetCoordinateDim(dm, dim));
739   switch (dim) {
740   case 1:
741     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(dm, lower, upper, faces));
742     break;
743   case 2:
744     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(dm, lower, upper, faces));
745     break;
746   case 3:
747     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(dm, lower, upper, faces));
748     break;
749   default:
750     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Dimension not supported: %" PetscInt_FMT, dim);
751   }
752   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
753   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
754   PetscFunctionReturn(PETSC_SUCCESS);
755 }
756 
757 /*@C
758   DMPlexCreateBoxSurfaceMesh - Creates a mesh on the surface of the tensor product of unit intervals (box) using tensor cells (hexahedra).
759 
760   Collective
761 
762   Input Parameters:
763 + comm        - The communicator for the `DM` object
764 . dim         - The spatial dimension of the box, so the resulting mesh is has dimension `dim`-1
765 . faces       - Number of faces per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
766 . lower       - The lower left corner, or `NULL` for (0, 0, 0)
767 . upper       - The upper right corner, or `NULL` for (1, 1, 1)
768 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
769 
770   Output Parameter:
771 . dm - The `DM` object
772 
773   Level: beginner
774 
775 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateBoxMesh()`, `DMPlexCreateFromFile()`, `DMSetType()`, `DMCreate()`
776 @*/
777 PetscErrorCode DMPlexCreateBoxSurfaceMesh(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate, DM *dm)
778 {
779   PetscInt  fac[3] = {1, 1, 1};
780   PetscReal low[3] = {0, 0, 0};
781   PetscReal upp[3] = {1, 1, 1};
782 
783   PetscFunctionBegin;
784   PetscCall(DMCreate(comm, dm));
785   PetscCall(DMSetType(*dm, DMPLEX));
786   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(*dm, dim, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, interpolate));
787   PetscFunctionReturn(PETSC_SUCCESS);
788 }
789 
790 static PetscErrorCode DMPlexCreateLineMesh_Internal(DM dm, PetscInt segments, PetscReal lower, PetscReal upper, DMBoundaryType bd)
791 {
792   PetscInt     i, fStart, fEnd, numCells = 0, numVerts = 0;
793   PetscInt     numPoints[2], *coneSize, *cones, *coneOrientations;
794   PetscScalar *vertexCoords;
795   PetscReal    L, maxCell;
796   PetscBool    markerSeparate = PETSC_FALSE;
797   PetscInt     markerLeft = 1, faceMarkerLeft = 1;
798   PetscInt     markerRight = 1, faceMarkerRight = 2;
799   PetscBool    wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
800   PetscMPIInt  rank;
801 
802   PetscFunctionBegin;
803   PetscAssertPointer(dm, 1);
804 
805   PetscCall(DMSetDimension(dm, 1));
806   PetscCall(DMCreateLabel(dm, "marker"));
807   PetscCall(DMCreateLabel(dm, "Face Sets"));
808 
809   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
810   if (rank == 0) numCells = segments;
811   if (rank == 0) numVerts = segments + (wrap ? 0 : 1);
812 
813   numPoints[0] = numVerts;
814   numPoints[1] = numCells;
815   PetscCall(PetscMalloc4(numCells + numVerts, &coneSize, numCells * 2, &cones, numCells + numVerts, &coneOrientations, numVerts, &vertexCoords));
816   PetscCall(PetscArrayzero(coneOrientations, numCells + numVerts));
817   for (i = 0; i < numCells; ++i) coneSize[i] = 2;
818   for (i = 0; i < numVerts; ++i) coneSize[numCells + i] = 0;
819   for (i = 0; i < numCells; ++i) {
820     cones[2 * i]     = numCells + i % numVerts;
821     cones[2 * i + 1] = numCells + (i + 1) % numVerts;
822   }
823   for (i = 0; i < numVerts; ++i) vertexCoords[i] = lower + (upper - lower) * ((PetscReal)i / (PetscReal)numCells);
824   PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
825   PetscCall(PetscFree4(coneSize, cones, coneOrientations, vertexCoords));
826 
827   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
828   if (markerSeparate) {
829     markerLeft  = faceMarkerLeft;
830     markerRight = faceMarkerRight;
831   }
832   if (!wrap && rank == 0) {
833     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
834     PetscCall(DMSetLabelValue(dm, "marker", fStart, markerLeft));
835     PetscCall(DMSetLabelValue(dm, "marker", fEnd - 1, markerRight));
836     PetscCall(DMSetLabelValue(dm, "Face Sets", fStart, faceMarkerLeft));
837     PetscCall(DMSetLabelValue(dm, "Face Sets", fEnd - 1, faceMarkerRight));
838   }
839   if (wrap) {
840     L       = upper - lower;
841     maxCell = (PetscReal)1.1 * (L / (PetscReal)PetscMax(1, segments));
842     PetscCall(DMSetPeriodicity(dm, &maxCell, &lower, &L));
843   }
844   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
845   PetscFunctionReturn(PETSC_SUCCESS);
846 }
847 
848 // Creates "Face Sets" label based on the standard box labeling conventions
849 static PetscErrorCode DMPlexSetBoxLabel_Internal(DM dm, const DMBoundaryType periodicity[])
850 {
851   DM              cdm;
852   PetscSection    csection;
853   Vec             coordinates;
854   DMLabel         label;
855   IS              faces_is;
856   PetscInt        dim, num_face;
857   const PetscInt *faces;
858   PetscInt        faceMarkerBottom, faceMarkerTop, faceMarkerFront, faceMarkerBack, faceMarkerRight, faceMarkerLeft;
859 
860   PetscFunctionBeginUser;
861   PetscCall(DMGetDimension(dm, &dim));
862   PetscCheck((dim == 2) || (dim == 3), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "DMPlex box labeling only supports 2D and 3D meshes, received DM of dimension %" PetscInt_FMT, dim);
863   // Get Face Sets label
864   PetscCall(DMGetLabel(dm, "Face Sets", &label));
865   if (label) {
866     PetscCall(DMLabelReset(label));
867   } else {
868     PetscCall(DMCreateLabel(dm, "Face Sets"));
869     PetscCall(DMGetLabel(dm, "Face Sets", &label));
870   }
871   PetscCall(DMPlexMarkBoundaryFaces(dm, 1, label));
872   PetscCall(DMGetStratumIS(dm, "Face Sets", 1, &faces_is));
873   if (!faces_is) PetscFunctionReturn(PETSC_SUCCESS); // No faces on rank
874 
875   switch (dim) {
876   case 2:
877     faceMarkerTop    = 3;
878     faceMarkerBottom = 1;
879     faceMarkerRight  = 2;
880     faceMarkerLeft   = 4;
881     break;
882   case 3:
883     faceMarkerBottom = 1;
884     faceMarkerTop    = 2;
885     faceMarkerFront  = 3;
886     faceMarkerBack   = 4;
887     faceMarkerRight  = 5;
888     faceMarkerLeft   = 6;
889     break;
890   default:
891     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
892   }
893 
894   PetscCall(ISGetLocalSize(faces_is, &num_face));
895   PetscCall(ISGetIndices(faces_is, &faces));
896   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
897   PetscCall(DMGetCoordinateDM(dm, &cdm));
898   PetscCall(DMGetLocalSection(cdm, &csection));
899   for (PetscInt f = 0; f < num_face; ++f) {
900     PetscScalar *coords = NULL;
901     PetscInt     face = faces[f], flip = 1, label_value = -1, coords_size;
902 
903     { // Determine if orientation of face is flipped
904       PetscInt        num_cells_support, num_faces, start = -1;
905       const PetscInt *orients, *cell_faces, *cells;
906 
907       PetscCall(DMPlexGetSupport(dm, face, &cells));
908       PetscCall(DMPlexGetSupportSize(dm, face, &num_cells_support));
909       PetscCheck(num_cells_support == 1, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Expected one cell in support of exterior face, but got %" PetscInt_FMT " cells", num_cells_support);
910       PetscCall(DMPlexGetCone(dm, cells[0], &cell_faces));
911       PetscCall(DMPlexGetConeSize(dm, cells[0], &num_faces));
912       for (PetscInt i = 0; i < num_faces; i++) {
913         if (cell_faces[i] == face) start = i;
914       }
915       PetscCheck(start >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_CORRUPT, "Could not find face %" PetscInt_FMT " in cone of its support", face);
916       PetscCall(DMPlexGetConeOrientation(dm, cells[0], &orients));
917       if (orients[start] < 0) flip = -1;
918     }
919 
920     // Cannot use DMPlexComputeCellGeometryFVM() for high-order geometry, so must calculate normal vectors manually
921     // Use the vertices (depth 0) of coordinate DM to calculate normal vector
922     PetscCall(DMPlexVecGetClosureAtDepth_Internal(cdm, csection, coordinates, face, 0, &coords_size, &coords));
923     switch (dim) {
924     case 2: {
925       PetscScalar vec[2];
926 
927       for (PetscInt d = 0; d < dim; ++d) vec[d] = flip * (PetscRealPart(coords[1 * dim + d]) - PetscRealPart(coords[0 * dim + d]));
928       PetscScalar normal[] = {vec[1], -vec[0]};
929       if (PetscAbsScalar(normal[0]) > PetscAbsScalar(normal[1])) {
930         label_value = PetscRealPart(normal[0]) > 0 ? faceMarkerRight : faceMarkerLeft;
931       } else {
932         label_value = PetscRealPart(normal[1]) > 0 ? faceMarkerTop : faceMarkerBottom;
933       }
934     } break;
935     case 3: {
936       PetscScalar vec1[3], vec2[3], normal[3];
937 
938       for (PetscInt d = 0; d < dim; ++d) {
939         vec1[d] = PetscRealPart(coords[1 * dim + d]) - PetscRealPart(coords[0 * dim + d]);
940         vec2[d] = PetscRealPart(coords[2 * dim + d]) - PetscRealPart(coords[1 * dim + d]);
941       }
942 
943       // Calculate normal vector via cross-product
944       normal[0] = flip * ((vec1[1] * vec2[2]) - (vec1[2] * vec2[1]));
945       normal[1] = flip * ((vec1[2] * vec2[0]) - (vec1[0] * vec2[2]));
946       normal[2] = flip * ((vec1[0] * vec2[1]) - (vec1[1] * vec2[0]));
947 
948       if (PetscAbsScalar(normal[0]) > PetscAbsScalar(normal[1])) {
949         if (PetscAbsScalar(normal[0]) > PetscAbsScalar(normal[2])) {
950           label_value = PetscRealPart(normal[0]) > 0 ? faceMarkerRight : faceMarkerLeft;
951         } else {
952           label_value = PetscRealPart(normal[2]) > 0 ? faceMarkerTop : faceMarkerBottom;
953         }
954       } else {
955         if (PetscAbsScalar(normal[1]) > PetscAbsScalar(normal[2])) {
956           label_value = PetscRealPart(normal[1]) > 0 ? faceMarkerBack : faceMarkerFront;
957         } else {
958           label_value = PetscRealPart(normal[2]) > 0 ? faceMarkerTop : faceMarkerBottom;
959         }
960       }
961     } break;
962     }
963 
964     PetscInt previous_label_value; // always 1 due to DMPlexMarkBoundaryFaces call above
965     PetscCall(DMGetLabelValue(dm, "Face Sets", face, &previous_label_value));
966     PetscCall(DMClearLabelValue(dm, "Face Sets", face, previous_label_value));
967     PetscCall(DMSetLabelValue(dm, "Face Sets", face, label_value));
968     PetscCall(DMPlexVecRestoreClosure(cdm, csection, coordinates, face, &coords_size, &coords));
969   }
970   PetscCall(ISRestoreIndices(faces_is, &faces));
971   PetscCall(ISDestroy(&faces_is));
972 
973   // Create Isoperiodic SF from newly-created face labels
974   PetscSF     periodicsfs[3];
975   PetscInt    periodic_sf_index  = 0;
976   PetscScalar transform[3][4][4] = {{{0.}}};
977   for (PetscInt d = 0; d < dim; d++) {
978     IS              donor_is, periodic_is;
979     const PetscInt *donor_faces = NULL, *periodic_faces = NULL;
980     PetscInt        num_donor = 0, num_periodic = 0;
981     PetscSF         centroidsf;
982     PetscReal       donor_to_periodic_distance;
983     const PetscInt  face_pairings[2][3][2] = {
984       // 2D face pairings, {donor, periodic}
985       {{4, 2}, {1, 3}},
986       // 3D face pairings
987       {{5, 6}, {3, 4}, {1, 2}}
988     };
989 
990     if (periodicity[d] != DM_BOUNDARY_PERIODIC) continue;
991     {
992       // Compute centroidsf, which is the mapping from donor faces to periodic faces
993       // Matches the centroid of the faces together, ignoring the periodic direction component (which should not match between donor and periodic face)
994       PetscInt     coords_size, centroid_comps = dim - 1;
995       PetscScalar *coords = NULL;
996       PetscReal   *donor_centroids, *periodic_centroids;
997       PetscReal    loc_periodic[2] = {PETSC_MIN_REAL, PETSC_MIN_REAL}, loc_periodic_global[2]; // Location of donor (0) and periodic (1) faces in periodic direction
998 
999       PetscCall(DMGetStratumIS(dm, "Face Sets", face_pairings[dim - 2][d][0], &donor_is));
1000       PetscCall(DMGetStratumIS(dm, "Face Sets", face_pairings[dim - 2][d][1], &periodic_is));
1001       if (donor_is) {
1002         PetscCall(ISGetLocalSize(donor_is, &num_donor));
1003         PetscCall(ISGetIndices(donor_is, &donor_faces));
1004       }
1005       if (periodic_is) {
1006         PetscCall(ISGetLocalSize(periodic_is, &num_periodic));
1007         PetscCall(ISGetIndices(periodic_is, &periodic_faces));
1008       }
1009       PetscCall(PetscCalloc2(num_donor * centroid_comps, &donor_centroids, num_periodic * centroid_comps, &periodic_centroids));
1010       for (PetscInt f = 0; f < num_donor; f++) {
1011         PetscInt face = donor_faces[f], num_coords;
1012         PetscCall(DMPlexVecGetClosureAtDepth_Internal(cdm, csection, coordinates, face, 0, &coords_size, &coords));
1013         num_coords = coords_size / dim;
1014         for (PetscInt c = 0; c < num_coords; c++) {
1015           PetscInt comp_index = 0;
1016           loc_periodic[0]     = PetscRealPart(coords[c * dim + d]);
1017           for (PetscInt i = 0; i < dim; i++) {
1018             if (i == d) continue; // Periodic direction not used for centroid calculation
1019             donor_centroids[f * centroid_comps + comp_index] += PetscRealPart(coords[c * dim + i]) / num_coords;
1020             comp_index++;
1021           }
1022         }
1023         PetscCall(DMPlexVecRestoreClosure(cdm, csection, coordinates, face, &coords_size, &coords));
1024       }
1025 
1026       for (PetscInt f = 0; f < num_periodic; f++) {
1027         PetscInt face = periodic_faces[f], num_coords;
1028         PetscCall(DMPlexVecGetClosureAtDepth_Internal(cdm, csection, coordinates, face, 0, &coords_size, &coords));
1029         num_coords = coords_size / dim;
1030         for (PetscInt c = 0; c < num_coords; c++) {
1031           PetscInt comp_index = 0;
1032           loc_periodic[1]     = PetscRealPart(coords[c * dim + d]);
1033           for (PetscInt i = 0; i < dim; i++) {
1034             if (i == d) continue; // Periodic direction not used for centroid calculation
1035             periodic_centroids[f * centroid_comps + comp_index] += PetscRealPart(coords[c * dim + i]) / num_coords;
1036             comp_index++;
1037           }
1038         }
1039         PetscCall(DMPlexVecRestoreClosure(cdm, csection, coordinates, face, &coords_size, &coords));
1040       }
1041       PetscCallMPI(MPIU_Allreduce(loc_periodic, loc_periodic_global, 2, MPIU_REAL, MPIU_MAX, PetscObjectComm((PetscObject)dm)));
1042       donor_to_periodic_distance = loc_periodic_global[1] - loc_periodic_global[0];
1043 
1044       PetscCall(PetscSFCreate(PetscObjectComm((PetscObject)dm), &centroidsf));
1045       PetscCall(PetscSFSetGraphFromCoordinates(centroidsf, num_donor, num_periodic, centroid_comps, 1e-10, donor_centroids, periodic_centroids));
1046       PetscCall(PetscSFViewFromOptions(centroidsf, NULL, "-dm_plex_box_label_centroid_sf_view"));
1047       PetscCall(PetscFree2(donor_centroids, periodic_centroids));
1048     }
1049 
1050     { // Create Isoperiodic SF using centroidsSF
1051       PetscInt           pStart, pEnd;
1052       PetscInt          *leaf_faces;
1053       const PetscSFNode *firemote;
1054       PetscSFNode       *isoperiodic_leaves;
1055 
1056       PetscCall(PetscMalloc1(num_periodic, &leaf_faces));
1057       PetscCall(PetscSFBcastBegin(centroidsf, MPIU_INT, donor_faces, leaf_faces, MPI_REPLACE));
1058       PetscCall(PetscSFBcastEnd(centroidsf, MPIU_INT, donor_faces, leaf_faces, MPI_REPLACE));
1059 
1060       PetscCall(PetscMalloc1(num_periodic, &isoperiodic_leaves));
1061       PetscCall(PetscSFGetGraph(centroidsf, NULL, NULL, NULL, &firemote));
1062       for (PetscInt l = 0; l < num_periodic; ++l) {
1063         isoperiodic_leaves[l].index = leaf_faces[l];
1064         isoperiodic_leaves[l].rank  = firemote[l].rank;
1065       }
1066 
1067       PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
1068       PetscCall(PetscSFCreate(PetscObjectComm((PetscObject)dm), &periodicsfs[periodic_sf_index]));
1069       PetscCall(PetscSFSetGraph(periodicsfs[periodic_sf_index], pEnd - pStart, num_periodic, (PetscInt *)periodic_faces, PETSC_COPY_VALUES, isoperiodic_leaves, PETSC_OWN_POINTER));
1070       PetscCall(PetscSFViewFromOptions(periodicsfs[periodic_sf_index], NULL, "-dm_plex_box_label_periodic_sf_view"));
1071       PetscCall(PetscFree(leaf_faces));
1072     }
1073 
1074     transform[periodic_sf_index][0][0] = 1;
1075     transform[periodic_sf_index][1][1] = 1;
1076     transform[periodic_sf_index][2][2] = 1;
1077     transform[periodic_sf_index][3][3] = 1;
1078     transform[periodic_sf_index][d][3] = donor_to_periodic_distance;
1079 
1080     periodic_sf_index++;
1081     PetscCall(PetscSFDestroy(&centroidsf));
1082     if (donor_is) {
1083       PetscCall(ISRestoreIndices(donor_is, &donor_faces));
1084       PetscCall(ISDestroy(&donor_is));
1085     }
1086     if (periodic_is) {
1087       PetscCall(ISRestoreIndices(periodic_is, &periodic_faces));
1088       PetscCall(ISDestroy(&periodic_is));
1089     }
1090     PetscCall(DMClearLabelStratum(dm, "Face Sets", face_pairings[dim - 2][d][0]));
1091     PetscCall(DMClearLabelStratum(dm, "Face Sets", face_pairings[dim - 2][d][1]));
1092   }
1093   PetscCall(DMPlexSetIsoperiodicFaceSF(dm, periodic_sf_index, periodicsfs));
1094   PetscCall(DMPlexSetIsoperiodicFaceTransform(dm, periodic_sf_index, (const PetscScalar *)transform));
1095   for (PetscInt p = 0; p < periodic_sf_index; p++) PetscCall(PetscSFDestroy(&periodicsfs[p]));
1096 
1097   { // Update coordinate DM with new Face Sets label
1098     DM      cdm;
1099     DMLabel oldFaceSets, newFaceSets;
1100     PetscCall(DMGetCoordinateDM(dm, &cdm));
1101     PetscCall(DMGetLabel(cdm, "Face Sets", &oldFaceSets));
1102     if (oldFaceSets) PetscCall(DMRemoveLabelBySelf(cdm, &oldFaceSets, PETSC_FALSE));
1103     PetscCall(DMLabelDuplicate(label, &newFaceSets));
1104     PetscCall(DMAddLabel(cdm, newFaceSets));
1105     PetscCall(DMLabelDestroy(&newFaceSets));
1106   }
1107   PetscFunctionReturn(PETSC_SUCCESS);
1108 }
1109 
1110 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1111 {
1112   DM      boundary, vol;
1113   DMLabel bdlabel;
1114 
1115   PetscFunctionBegin;
1116   PetscAssertPointer(dm, 1);
1117   for (PetscInt i = 0; i < dim; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
1118   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &boundary));
1119   PetscCall(DMSetType(boundary, DMPLEX));
1120   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(boundary, dim, faces, lower, upper, PETSC_FALSE));
1121   PetscCall(DMPlexGenerate(boundary, NULL, interpolate, &vol));
1122   PetscCall(DMGetLabel(vol, "marker", &bdlabel));
1123   if (bdlabel) PetscCall(DMPlexLabelComplete(vol, bdlabel));
1124   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_FALSE, vol));
1125   PetscCall(DMPlexReplace_Internal(dm, &vol));
1126   if (interpolate) {
1127     PetscCall(DMPlexInterpolateInPlace_Internal(dm));
1128     PetscCall(DMPlexSetBoxLabel_Internal(dm, periodicity));
1129   }
1130   PetscCall(DMDestroy(&boundary));
1131   PetscFunctionReturn(PETSC_SUCCESS);
1132 }
1133 
1134 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
1135 {
1136   DMLabel     cutLabel  = NULL;
1137   PetscInt    markerTop = 1, faceMarkerTop = 1;
1138   PetscInt    markerBottom = 1, faceMarkerBottom = 1;
1139   PetscInt    markerFront = 1, faceMarkerFront = 1;
1140   PetscInt    markerBack = 1, faceMarkerBack = 1;
1141   PetscInt    markerRight = 1, faceMarkerRight = 1;
1142   PetscInt    markerLeft = 1, faceMarkerLeft = 1;
1143   PetscInt    dim;
1144   PetscBool   markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
1145   PetscMPIInt rank;
1146 
1147   PetscFunctionBegin;
1148   PetscCall(DMGetDimension(dm, &dim));
1149   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1150   PetscCall(DMCreateLabel(dm, "marker"));
1151   PetscCall(DMCreateLabel(dm, "Face Sets"));
1152   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
1153   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST || bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST || bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
1154     if (cutMarker) {
1155       PetscCall(DMCreateLabel(dm, "periodic_cut"));
1156       PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
1157     }
1158   }
1159   switch (dim) {
1160   case 2:
1161     faceMarkerTop    = 3;
1162     faceMarkerBottom = 1;
1163     faceMarkerRight  = 2;
1164     faceMarkerLeft   = 4;
1165     break;
1166   case 3:
1167     faceMarkerBottom = 1;
1168     faceMarkerTop    = 2;
1169     faceMarkerFront  = 3;
1170     faceMarkerBack   = 4;
1171     faceMarkerRight  = 5;
1172     faceMarkerLeft   = 6;
1173     break;
1174   default:
1175     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
1176   }
1177   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
1178   if (markerSeparate) {
1179     markerBottom = faceMarkerBottom;
1180     markerTop    = faceMarkerTop;
1181     markerFront  = faceMarkerFront;
1182     markerBack   = faceMarkerBack;
1183     markerRight  = faceMarkerRight;
1184     markerLeft   = faceMarkerLeft;
1185   }
1186   {
1187     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
1188     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
1189     const PetscInt numZEdges    = rank == 0 ? edges[2] : 0;
1190     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0] + 1) : 0;
1191     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1] + 1) : 0;
1192     const PetscInt numZVertices = rank == 0 ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2] + 1) : 0;
1193     const PetscInt numCells     = numXEdges * numYEdges * numZEdges;
1194     const PetscInt numXFaces    = numYEdges * numZEdges;
1195     const PetscInt numYFaces    = numXEdges * numZEdges;
1196     const PetscInt numZFaces    = numXEdges * numYEdges;
1197     const PetscInt numTotXFaces = numXVertices * numXFaces;
1198     const PetscInt numTotYFaces = numYVertices * numYFaces;
1199     const PetscInt numTotZFaces = numZVertices * numZFaces;
1200     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
1201     const PetscInt numTotXEdges = numXEdges * numYVertices * numZVertices;
1202     const PetscInt numTotYEdges = numYEdges * numXVertices * numZVertices;
1203     const PetscInt numTotZEdges = numZEdges * numXVertices * numYVertices;
1204     const PetscInt numVertices  = numXVertices * numYVertices * numZVertices;
1205     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
1206     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
1207     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
1208     const PetscInt firstYFace   = firstXFace + numTotXFaces;
1209     const PetscInt firstZFace   = firstYFace + numTotYFaces;
1210     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
1211     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
1212     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
1213     Vec            coordinates;
1214     PetscSection   coordSection;
1215     PetscScalar   *coords;
1216     PetscInt       coordSize;
1217     PetscInt       v, vx, vy, vz;
1218     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
1219 
1220     PetscCall(DMPlexSetChart(dm, 0, numCells + numFaces + numEdges + numVertices));
1221     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
1222     for (f = firstXFace; f < firstXFace + numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
1223     for (e = firstXEdge; e < firstXEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
1224     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
1225     /* Build cells */
1226     for (fz = 0; fz < numZEdges; ++fz) {
1227       for (fy = 0; fy < numYEdges; ++fy) {
1228         for (fx = 0; fx < numXEdges; ++fx) {
1229           PetscInt cell  = (fz * numYEdges + fy) * numXEdges + fx;
1230           PetscInt faceB = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
1231           PetscInt faceT = firstZFace + (fy * numXEdges + fx) * numZVertices + ((fz + 1) % numZVertices);
1232           PetscInt faceF = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
1233           PetscInt faceK = firstYFace + (fz * numXEdges + fx) * numYVertices + ((fy + 1) % numYVertices);
1234           PetscInt faceL = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
1235           PetscInt faceR = firstXFace + (fz * numYEdges + fy) * numXVertices + ((fx + 1) % numXVertices);
1236           /* B,  T,  F,  K,  R,  L */
1237           PetscInt ornt[6] = {-2, 0, 0, -3, 0, -2}; /* ??? */
1238           PetscInt cone[6];
1239 
1240           /* no boundary twisting in 3D */
1241           cone[0] = faceB;
1242           cone[1] = faceT;
1243           cone[2] = faceF;
1244           cone[3] = faceK;
1245           cone[4] = faceR;
1246           cone[5] = faceL;
1247           PetscCall(DMPlexSetCone(dm, cell, cone));
1248           PetscCall(DMPlexSetConeOrientation(dm, cell, ornt));
1249           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
1250           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
1251           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
1252         }
1253       }
1254     }
1255     /* Build x faces */
1256     for (fz = 0; fz < numZEdges; ++fz) {
1257       for (fy = 0; fy < numYEdges; ++fy) {
1258         for (fx = 0; fx < numXVertices; ++fx) {
1259           PetscInt face    = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
1260           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
1261           PetscInt edgeR   = firstZEdge + (((fy + 1) % numYVertices) * numXVertices + fx) * numZEdges + fz;
1262           PetscInt edgeB   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
1263           PetscInt edgeT   = firstYEdge + (((fz + 1) % numZVertices) * numXVertices + fx) * numYEdges + fy;
1264           PetscInt ornt[4] = {0, 0, -1, -1};
1265           PetscInt cone[4];
1266 
1267           if (dim == 3) {
1268             /* markers */
1269             if (bdX != DM_BOUNDARY_PERIODIC) {
1270               if (fx == numXVertices - 1) {
1271                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight));
1272                 PetscCall(DMSetLabelValue(dm, "marker", face, markerRight));
1273               } else if (fx == 0) {
1274                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft));
1275                 PetscCall(DMSetLabelValue(dm, "marker", face, markerLeft));
1276               }
1277             }
1278           }
1279           cone[0] = edgeB;
1280           cone[1] = edgeR;
1281           cone[2] = edgeT;
1282           cone[3] = edgeL;
1283           PetscCall(DMPlexSetCone(dm, face, cone));
1284           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1285         }
1286       }
1287     }
1288     /* Build y faces */
1289     for (fz = 0; fz < numZEdges; ++fz) {
1290       for (fx = 0; fx < numXEdges; ++fx) {
1291         for (fy = 0; fy < numYVertices; ++fy) {
1292           PetscInt face    = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
1293           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
1294           PetscInt edgeR   = firstZEdge + (fy * numXVertices + ((fx + 1) % numXVertices)) * numZEdges + fz;
1295           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1296           PetscInt edgeT   = firstXEdge + (((fz + 1) % numZVertices) * numYVertices + fy) * numXEdges + fx;
1297           PetscInt ornt[4] = {0, 0, -1, -1};
1298           PetscInt cone[4];
1299 
1300           if (dim == 3) {
1301             /* markers */
1302             if (bdY != DM_BOUNDARY_PERIODIC) {
1303               if (fy == numYVertices - 1) {
1304                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack));
1305                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBack));
1306               } else if (fy == 0) {
1307                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront));
1308                 PetscCall(DMSetLabelValue(dm, "marker", face, markerFront));
1309               }
1310             }
1311           }
1312           cone[0] = edgeB;
1313           cone[1] = edgeR;
1314           cone[2] = edgeT;
1315           cone[3] = edgeL;
1316           PetscCall(DMPlexSetCone(dm, face, cone));
1317           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1318         }
1319       }
1320     }
1321     /* Build z faces */
1322     for (fy = 0; fy < numYEdges; ++fy) {
1323       for (fx = 0; fx < numXEdges; ++fx) {
1324         for (fz = 0; fz < numZVertices; fz++) {
1325           PetscInt face    = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
1326           PetscInt edgeL   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
1327           PetscInt edgeR   = firstYEdge + (fz * numXVertices + ((fx + 1) % numXVertices)) * numYEdges + fy;
1328           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1329           PetscInt edgeT   = firstXEdge + (fz * numYVertices + ((fy + 1) % numYVertices)) * numXEdges + fx;
1330           PetscInt ornt[4] = {0, 0, -1, -1};
1331           PetscInt cone[4];
1332 
1333           if (dim == 2) {
1334             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges - 1) {
1335               edgeR += numYEdges - 1 - 2 * fy;
1336               ornt[1] = -1;
1337             }
1338             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges - 1) {
1339               edgeT += numXEdges - 1 - 2 * fx;
1340               ornt[2] = 0;
1341             }
1342             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1343             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1344           } else {
1345             /* markers */
1346             if (bdZ != DM_BOUNDARY_PERIODIC) {
1347               if (fz == numZVertices - 1) {
1348                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop));
1349                 PetscCall(DMSetLabelValue(dm, "marker", face, markerTop));
1350               } else if (fz == 0) {
1351                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom));
1352                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBottom));
1353               }
1354             }
1355           }
1356           cone[0] = edgeB;
1357           cone[1] = edgeR;
1358           cone[2] = edgeT;
1359           cone[3] = edgeL;
1360           PetscCall(DMPlexSetCone(dm, face, cone));
1361           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1362         }
1363       }
1364     }
1365     /* Build Z edges*/
1366     for (vy = 0; vy < numYVertices; vy++) {
1367       for (vx = 0; vx < numXVertices; vx++) {
1368         for (ez = 0; ez < numZEdges; ez++) {
1369           const PetscInt edge    = firstZEdge + (vy * numXVertices + vx) * numZEdges + ez;
1370           const PetscInt vertexB = firstVertex + (ez * numYVertices + vy) * numXVertices + vx;
1371           const PetscInt vertexT = firstVertex + (((ez + 1) % numZVertices) * numYVertices + vy) * numXVertices + vx;
1372           PetscInt       cone[2];
1373 
1374           cone[0] = vertexB;
1375           cone[1] = vertexT;
1376           PetscCall(DMPlexSetCone(dm, edge, cone));
1377           if (dim == 3) {
1378             if (bdX != DM_BOUNDARY_PERIODIC) {
1379               if (vx == numXVertices - 1) {
1380                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1381                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1382                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1383               } else if (vx == 0) {
1384                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1385                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1386                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1387               }
1388             }
1389             if (bdY != DM_BOUNDARY_PERIODIC) {
1390               if (vy == numYVertices - 1) {
1391                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1392                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1393                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1394               } else if (vy == 0) {
1395                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1396                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1397                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1398               }
1399             }
1400           }
1401         }
1402       }
1403     }
1404     /* Build Y edges*/
1405     for (vz = 0; vz < numZVertices; vz++) {
1406       for (vx = 0; vx < numXVertices; vx++) {
1407         for (ey = 0; ey < numYEdges; ey++) {
1408           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges - 1) ? (numXVertices - vx - 1) : (vz * numYVertices + ((ey + 1) % numYVertices)) * numXVertices + vx;
1409           const PetscInt edge    = firstYEdge + (vz * numXVertices + vx) * numYEdges + ey;
1410           const PetscInt vertexF = firstVertex + (vz * numYVertices + ey) * numXVertices + vx;
1411           const PetscInt vertexK = firstVertex + nextv;
1412           PetscInt       cone[2];
1413 
1414           cone[0] = vertexF;
1415           cone[1] = vertexK;
1416           PetscCall(DMPlexSetCone(dm, edge, cone));
1417           if (dim == 2) {
1418             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1419               if (vx == numXVertices - 1) {
1420                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight));
1421                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1422                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1423                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1424               } else if (vx == 0) {
1425                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft));
1426                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1427                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1428                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1429               }
1430             } else {
1431               if (vx == 0 && cutLabel) {
1432                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1433                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1434                 if (ey == numYEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1435               }
1436             }
1437           } else {
1438             if (bdX != DM_BOUNDARY_PERIODIC) {
1439               if (vx == numXVertices - 1) {
1440                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1441                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1442                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1443               } else if (vx == 0) {
1444                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1445                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1446                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1447               }
1448             }
1449             if (bdZ != DM_BOUNDARY_PERIODIC) {
1450               if (vz == numZVertices - 1) {
1451                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1452                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1453                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1454               } else if (vz == 0) {
1455                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1456                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1457                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1458               }
1459             }
1460           }
1461         }
1462       }
1463     }
1464     /* Build X edges*/
1465     for (vz = 0; vz < numZVertices; vz++) {
1466       for (vy = 0; vy < numYVertices; vy++) {
1467         for (ex = 0; ex < numXEdges; ex++) {
1468           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges - 1) ? (numYVertices - vy - 1) * numXVertices : (vz * numYVertices + vy) * numXVertices + (ex + 1) % numXVertices;
1469           const PetscInt edge    = firstXEdge + (vz * numYVertices + vy) * numXEdges + ex;
1470           const PetscInt vertexL = firstVertex + (vz * numYVertices + vy) * numXVertices + ex;
1471           const PetscInt vertexR = firstVertex + nextv;
1472           PetscInt       cone[2];
1473 
1474           cone[0] = vertexL;
1475           cone[1] = vertexR;
1476           PetscCall(DMPlexSetCone(dm, edge, cone));
1477           if (dim == 2) {
1478             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1479               if (vy == numYVertices - 1) {
1480                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop));
1481                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1482                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1483                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1484               } else if (vy == 0) {
1485                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom));
1486                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1487                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1488                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1489               }
1490             } else {
1491               if (vy == 0 && cutLabel) {
1492                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1493                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1494                 if (ex == numXEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1495               }
1496             }
1497           } else {
1498             if (bdY != DM_BOUNDARY_PERIODIC) {
1499               if (vy == numYVertices - 1) {
1500                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1501                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1502                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1503               } else if (vy == 0) {
1504                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1505                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1506                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1507               }
1508             }
1509             if (bdZ != DM_BOUNDARY_PERIODIC) {
1510               if (vz == numZVertices - 1) {
1511                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1512                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1513                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1514               } else if (vz == 0) {
1515                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1516                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1517                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1518               }
1519             }
1520           }
1521         }
1522       }
1523     }
1524     PetscCall(DMPlexSymmetrize(dm));
1525     PetscCall(DMPlexStratify(dm));
1526     /* Build coordinates */
1527     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1528     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1529     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1530     PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVertices));
1531     for (v = firstVertex; v < firstVertex + numVertices; ++v) {
1532       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1533       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1534     }
1535     PetscCall(PetscSectionSetUp(coordSection));
1536     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1537     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1538     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1539     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1540     PetscCall(VecSetBlockSize(coordinates, dim));
1541     PetscCall(VecSetType(coordinates, VECSTANDARD));
1542     PetscCall(VecGetArray(coordinates, &coords));
1543     for (vz = 0; vz < numZVertices; ++vz) {
1544       for (vy = 0; vy < numYVertices; ++vy) {
1545         for (vx = 0; vx < numXVertices; ++vx) {
1546           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * vx;
1547           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * vy;
1548           if (dim == 3) coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 2] = lower[2] + ((upper[2] - lower[2]) / numZEdges) * vz;
1549         }
1550       }
1551     }
1552     PetscCall(VecRestoreArray(coordinates, &coords));
1553     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1554     PetscCall(VecDestroy(&coordinates));
1555   }
1556   PetscFunctionReturn(PETSC_SUCCESS);
1557 }
1558 
1559 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1560 {
1561   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1562   PetscInt       fac[3] = {0, 0, 0}, d;
1563 
1564   PetscFunctionBegin;
1565   PetscAssertPointer(dm, 1);
1566   PetscValidLogicalCollectiveInt(dm, dim, 2);
1567   PetscCall(DMSetDimension(dm, dim));
1568   for (d = 0; d < dim; ++d) {
1569     fac[d] = faces[d];
1570     bdt[d] = periodicity[d];
1571   }
1572   PetscCall(DMPlexCreateCubeMesh_Internal(dm, lower, upper, fac, bdt[0], bdt[1], bdt[2]));
1573   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST || periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST || (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
1574     PetscReal L[3]       = {-1., -1., 0.};
1575     PetscReal maxCell[3] = {-1., -1., 0.};
1576 
1577     for (d = 0; d < dim; ++d) {
1578       if (periodicity[d] != DM_BOUNDARY_NONE) {
1579         L[d]       = upper[d] - lower[d];
1580         maxCell[d] = 1.1 * (L[d] / PetscMax(1, faces[d]));
1581       }
1582     }
1583     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1584   }
1585   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
1586   PetscFunctionReturn(PETSC_SUCCESS);
1587 }
1588 
1589 static PetscErrorCode DMPlexCreateBoxMesh_Internal(DM dm, DMPlexShape shape, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1590 {
1591   PetscFunctionBegin;
1592   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
1593   if (shape == DM_SHAPE_ZBOX) PetscCall(DMPlexCreateBoxMesh_Tensor_SFC_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1594   else if (dim == 1) PetscCall(DMPlexCreateLineMesh_Internal(dm, faces[0], lower[0], upper[0], periodicity[0]));
1595   else if (simplex) PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1596   else PetscCall(DMPlexCreateBoxMesh_Tensor_Internal(dm, dim, faces, lower, upper, periodicity));
1597   if (!interpolate && dim > 1 && !simplex) {
1598     DM udm;
1599 
1600     PetscCall(DMPlexUninterpolate(dm, &udm));
1601     PetscCall(DMPlexCopyCoordinates(dm, udm));
1602     PetscCall(DMPlexReplace_Internal(dm, &udm));
1603   }
1604   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
1605   PetscFunctionReturn(PETSC_SUCCESS);
1606 }
1607 
1608 /*@
1609   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
1610 
1611   Collective
1612 
1613   Input Parameters:
1614 + comm               - The communicator for the `DM` object
1615 . dim                - The spatial dimension
1616 . simplex            - `PETSC_TRUE` for simplices, `PETSC_FALSE` for tensor cells
1617 . faces              - Number of faces per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1618 . lower              - The lower left corner, or `NULL` for (0, 0, 0)
1619 . upper              - The upper right corner, or `NULL` for (1, 1, 1)
1620 . periodicity        - The boundary type for the X,Y,Z direction, or `NULL` for `DM_BOUNDARY_NONE`
1621 . interpolate        - Flag to create intermediate mesh pieces (edges, faces)
1622 . localizationHeight - Flag to localize edges and faces in addition to cells; only significant for periodic meshes
1623 - sparseLocalize     - Flag to localize coordinates only for cells near the periodic boundary; only significant for periodic meshes
1624 
1625   Output Parameter:
1626 . dm - The `DM` object
1627 
1628   Level: beginner
1629 
1630   Note:
1631   To customize this mesh using options, use
1632 .vb
1633   DMCreate(comm, &dm);
1634   DMSetType(dm, DMPLEX);
1635   DMSetFromOptions(dm);
1636 .ve
1637   and use the options in `DMSetFromOptions()`.
1638 
1639   Here is the numbering returned for 2 faces in each direction for tensor cells\:
1640 .vb
1641  10---17---11---18----12
1642   |         |         |
1643   |         |         |
1644  20    2   22    3    24
1645   |         |         |
1646   |         |         |
1647   7---15----8---16----9
1648   |         |         |
1649   |         |         |
1650  19    0   21    1   23
1651   |         |         |
1652   |         |         |
1653   4---13----5---14----6
1654 .ve
1655   and for simplicial cells
1656 .vb
1657  14----8---15----9----16
1658   |\     5  |\      7 |
1659   | \       | \       |
1660  13   2    14    3    15
1661   | 4   \   | 6   \   |
1662   |       \ |       \ |
1663  11----6---12----7----13
1664   |\        |\        |
1665   | \    1  | \     3 |
1666  10   0    11    1    12
1667   | 0   \   | 2   \   |
1668   |       \ |       \ |
1669   8----4----9----5----10
1670 .ve
1671 
1672 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
1673 @*/
1674 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, PetscInt localizationHeight, PetscBool sparseLocalize, DM *dm)
1675 {
1676   PetscInt       fac[3] = {1, 1, 1};
1677   PetscReal      low[3] = {0, 0, 0};
1678   PetscReal      upp[3] = {1, 1, 1};
1679   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1680 
1681   PetscFunctionBegin;
1682   PetscCall(DMCreate(comm, dm));
1683   PetscCall(DMSetType(*dm, DMPLEX));
1684   PetscCall(DMPlexCreateBoxMesh_Internal(*dm, DM_SHAPE_BOX, dim, simplex, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt, interpolate));
1685   if (periodicity) {
1686     DM cdm;
1687 
1688     PetscCall(DMGetCoordinateDM(*dm, &cdm));
1689     PetscCall(DMPlexSetMaxProjectionHeight(cdm, localizationHeight));
1690     PetscCall(DMSetSparseLocalize(*dm, sparseLocalize));
1691     PetscCall(DMLocalizeCoordinates(*dm));
1692   }
1693   PetscFunctionReturn(PETSC_SUCCESS);
1694 }
1695 
1696 static PetscErrorCode DMPlexCreateWedgeBoxMesh_Internal(DM dm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1697 {
1698   DM       bdm, vol;
1699   PetscInt i;
1700 
1701   PetscFunctionBegin;
1702   // TODO Now we can support periodicity
1703   for (i = 0; i < 3; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity not yet supported");
1704   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &bdm));
1705   PetscCall(DMSetType(bdm, DMPLEX));
1706   PetscCall(DMSetDimension(bdm, 2));
1707   PetscCall(PetscLogEventBegin(DMPLEX_Generate, bdm, 0, 0, 0));
1708   PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(bdm, 2, faces, lower, upper, periodicity, PETSC_TRUE));
1709   PetscCall(DMPlexExtrude(bdm, faces[2], upper[2] - lower[2], PETSC_TRUE, PETSC_FALSE, PETSC_FALSE, NULL, NULL, &vol));
1710   PetscCall(PetscLogEventEnd(DMPLEX_Generate, bdm, 0, 0, 0));
1711   PetscCall(DMDestroy(&bdm));
1712   PetscCall(DMPlexReplace_Internal(dm, &vol));
1713   if (lower[2] != 0.0) {
1714     Vec          v;
1715     PetscScalar *x;
1716     PetscInt     cDim, n;
1717 
1718     PetscCall(DMGetCoordinatesLocal(dm, &v));
1719     PetscCall(VecGetBlockSize(v, &cDim));
1720     PetscCall(VecGetLocalSize(v, &n));
1721     PetscCall(VecGetArray(v, &x));
1722     x += cDim;
1723     for (i = 0; i < n; i += cDim) x[i] += lower[2];
1724     PetscCall(VecRestoreArray(v, &x));
1725     PetscCall(DMSetCoordinatesLocal(dm, v));
1726   }
1727   PetscFunctionReturn(PETSC_SUCCESS);
1728 }
1729 
1730 /*@
1731   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tessellating the (x,y) plane and extruding in the third direction using wedge cells.
1732 
1733   Collective
1734 
1735   Input Parameters:
1736 + comm        - The communicator for the `DM` object
1737 . faces       - Number of faces per dimension, or `NULL` for (1, 1, 1)
1738 . lower       - The lower left corner, or `NULL` for (0, 0, 0)
1739 . upper       - The upper right corner, or `NULL` for (1, 1, 1)
1740 . periodicity - The boundary type for the X,Y,Z direction, or `NULL` for `DM_BOUNDARY_NONE`
1741 . orderHeight - If `PETSC_TRUE`, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
1742 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
1743 
1744   Output Parameter:
1745 . dm - The `DM` object
1746 
1747   Level: beginner
1748 
1749 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateWedgeCylinderMesh()`, `DMExtrude()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
1750 @*/
1751 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool orderHeight, PetscBool interpolate, DM *dm)
1752 {
1753   PetscInt       fac[3] = {1, 1, 1};
1754   PetscReal      low[3] = {0, 0, 0};
1755   PetscReal      upp[3] = {1, 1, 1};
1756   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1757 
1758   PetscFunctionBegin;
1759   PetscCall(DMCreate(comm, dm));
1760   PetscCall(DMSetType(*dm, DMPLEX));
1761   PetscCall(DMPlexCreateWedgeBoxMesh_Internal(*dm, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt));
1762   if (!interpolate) {
1763     DM udm;
1764 
1765     PetscCall(DMPlexUninterpolate(*dm, &udm));
1766     PetscCall(DMPlexReplace_Internal(*dm, &udm));
1767   }
1768   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
1769   PetscFunctionReturn(PETSC_SUCCESS);
1770 }
1771 
1772 /*
1773   DMPlexTensorPointLexicographic_Private - Returns all tuples of size 'len' with nonnegative integers that are all less than or equal to 'max' for that dimension.
1774 
1775   Input Parameters:
1776 + len - The length of the tuple
1777 . max - The maximum for each dimension, so values are in [0, max)
1778 - tup - A tuple of length len+1: tup[len] > 0 indicates a stopping condition
1779 
1780   Output Parameter:
1781 . tup - A tuple of `len` integers whose entries are at most `max`
1782 
1783   Level: developer
1784 
1785   Note:
1786   Ordering is lexicographic with lowest index as least significant in ordering.
1787   e.g. for len == 2 and max == 2, this will return, in order, {0,0}, {1,0}, {2,0}, {0,1}, {1,1}, {2,1}, {0,2}, {1,2}, {2,2}.
1788 
1789 .seealso: PetscDualSpaceTensorPointLexicographic_Internal(), PetscDualSpaceLatticePointLexicographic_Internal()
1790 */
1791 static PetscErrorCode DMPlexTensorPointLexicographic_Private(PetscInt len, const PetscInt max[], PetscInt tup[])
1792 {
1793   PetscInt i;
1794 
1795   PetscFunctionBegin;
1796   for (i = 0; i < len; ++i) {
1797     if (tup[i] < max[i] - 1) {
1798       break;
1799     } else {
1800       tup[i] = 0;
1801     }
1802   }
1803   if (i == len) tup[i - 1] = max[i - 1];
1804   else ++tup[i];
1805   PetscFunctionReturn(PETSC_SUCCESS);
1806 }
1807 
1808 static PetscInt TupleToIndex_Private(PetscInt len, const PetscInt max[], const PetscInt tup[])
1809 {
1810   PetscInt i, idx = tup[len - 1];
1811 
1812   for (i = len - 2; i >= 0; --i) {
1813     idx *= max[i];
1814     idx += tup[i];
1815   }
1816   return idx;
1817 }
1818 
1819 static PetscErrorCode DMPlexCreateHypercubicMesh_Internal(DM dm, PetscInt dim, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], const DMBoundaryType bd[])
1820 {
1821   Vec          coordinates;
1822   PetscSection coordSection;
1823   DMLabel      cutLabel    = NULL;
1824   PetscBool    cutMarker   = PETSC_FALSE;
1825   PetscBool    periodic    = PETSC_FALSE;
1826   PetscInt     numCells    = 1, c;
1827   PetscInt     numVertices = 1, v;
1828   PetscScalar *coords;
1829   PetscInt    *vertices, *vert, *vtmp, *supp, cone[2];
1830   PetscInt     d, e, cell = 0, coordSize;
1831   PetscMPIInt  rank;
1832 
1833   PetscFunctionBegin;
1834   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1835   PetscCall(DMSetDimension(dm, dim));
1836   PetscCall(PetscCalloc4(dim, &vertices, dim, &vert, dim, &vtmp, 2 * dim, &supp));
1837   PetscCall(DMCreateLabel(dm, "marker"));
1838   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
1839   for (d = 0; d < dim; ++d) periodic = (periodic || bd[d] == DM_BOUNDARY_PERIODIC) ? PETSC_TRUE : PETSC_FALSE;
1840   if (periodic && cutMarker) {
1841     PetscCall(DMCreateLabel(dm, "periodic_cut"));
1842     PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
1843   }
1844   for (d = 0; d < dim; ++d) PetscCheck(bd[d] == DM_BOUNDARY_PERIODIC, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Hypercubic mesh must be periodic now");
1845   for (d = 0; d < dim; ++d) {
1846     vertices[d] = edges[d];
1847     numVertices *= vertices[d];
1848   }
1849   numCells = numVertices * dim;
1850   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
1851   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, 2));
1852   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetSupportSize(dm, v, 2 * dim));
1853   /* TODO Loop over boundary and reset support sizes */
1854   PetscCall(DMSetUp(dm)); /* Allocate space for cones and supports */
1855   /* Build cell cones and vertex supports */
1856   PetscCall(DMCreateLabel(dm, "celltype"));
1857   while (vert[dim - 1] < vertices[dim - 1]) {
1858     const PetscInt vertex = TupleToIndex_Private(dim, vertices, vert) + numCells;
1859     PetscInt       s      = 0;
1860 
1861     PetscCall(PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT ":", vertex));
1862     for (d = 0; d < dim; ++d) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, vert[d]));
1863     PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
1864     PetscCall(DMPlexSetCellType(dm, vertex, DM_POLYTOPE_POINT));
1865     for (d = 0; d < dim; ++d) {
1866       for (e = 0; e < dim; ++e) vtmp[e] = vert[e];
1867       vtmp[d] = (vert[d] + 1) % vertices[d];
1868       cone[0] = vertex;
1869       cone[1] = TupleToIndex_Private(dim, vertices, vtmp) + numCells;
1870       PetscCall(PetscPrintf(PETSC_COMM_SELF, "  Vertex %" PetscInt_FMT ":", cone[1]));
1871       for (e = 0; e < dim; ++e) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, vtmp[e]));
1872       PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
1873       PetscCall(DMPlexSetCone(dm, cell, cone));
1874       PetscCall(DMPlexSetCellType(dm, cell, DM_POLYTOPE_SEGMENT));
1875       PetscCall(PetscPrintf(PETSC_COMM_SELF, "  Edge %" PetscInt_FMT " (%" PetscInt_FMT " %" PetscInt_FMT ")\n", cell, cone[0], cone[1]));
1876       ++cell;
1877     }
1878     for (d = 0; d < dim; ++d) {
1879       for (e = 0; e < dim; ++e) vtmp[e] = vert[e];
1880       vtmp[d]   = (vert[d] + vertices[d] - 1) % vertices[d];
1881       supp[s++] = TupleToIndex_Private(dim, vertices, vtmp) * dim + d;
1882       supp[s++] = (vertex - numCells) * dim + d;
1883       PetscCall(DMPlexSetSupport(dm, vertex, supp));
1884     }
1885     PetscCall(DMPlexTensorPointLexicographic_Private(dim, vertices, vert));
1886   }
1887   PetscCall(DMPlexStratify(dm));
1888   /* Build coordinates */
1889   PetscCall(DMGetCoordinateSection(dm, &coordSection));
1890   PetscCall(PetscSectionSetNumFields(coordSection, 1));
1891   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1892   PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
1893   for (v = numCells; v < numCells + numVertices; ++v) {
1894     PetscCall(PetscSectionSetDof(coordSection, v, dim));
1895     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1896   }
1897   PetscCall(PetscSectionSetUp(coordSection));
1898   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1899   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1900   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1901   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1902   PetscCall(VecSetBlockSize(coordinates, dim));
1903   PetscCall(VecSetType(coordinates, VECSTANDARD));
1904   PetscCall(VecGetArray(coordinates, &coords));
1905   for (d = 0; d < dim; ++d) vert[d] = 0;
1906   while (vert[dim - 1] < vertices[dim - 1]) {
1907     const PetscInt vertex = TupleToIndex_Private(dim, vertices, vert);
1908 
1909     for (d = 0; d < dim; ++d) coords[vertex * dim + d] = lower[d] + ((upper[d] - lower[d]) / vertices[d]) * vert[d];
1910     PetscCall(PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT ":", vertex));
1911     for (d = 0; d < dim; ++d) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, vert[d]));
1912     for (d = 0; d < dim; ++d) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %g", (double)PetscRealPart(coords[vertex * dim + d])));
1913     PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
1914     PetscCall(DMPlexTensorPointLexicographic_Private(dim, vertices, vert));
1915   }
1916   PetscCall(VecRestoreArray(coordinates, &coords));
1917   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1918   PetscCall(VecDestroy(&coordinates));
1919   PetscCall(PetscFree4(vertices, vert, vtmp, supp));
1920   //PetscCall(DMSetPeriodicity(dm, NULL, lower, upper));
1921   // Attach the extent
1922   {
1923     PetscContainer c;
1924     PetscInt      *extent;
1925 
1926     PetscCall(PetscMalloc1(dim, &extent));
1927     for (PetscInt d = 0; d < dim; ++d) extent[d] = edges[d];
1928     PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &c));
1929     PetscCall(PetscContainerSetCtxDestroy(c, PetscCtxDestroyDefault));
1930     PetscCall(PetscContainerSetPointer(c, extent));
1931     PetscCall(PetscObjectCompose((PetscObject)dm, "_extent", (PetscObject)c));
1932     PetscCall(PetscContainerDestroy(&c));
1933   }
1934   PetscFunctionReturn(PETSC_SUCCESS);
1935 }
1936 
1937 /*@C
1938   DMPlexCreateHypercubicMesh - Creates a periodic mesh on the tensor product of unit intervals using only vertices and edges.
1939 
1940   Collective
1941 
1942   Input Parameters:
1943 + comm  - The communicator for the DM object
1944 . dim   - The spatial dimension
1945 . edges - Number of edges per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1946 . lower - The lower left corner, or `NULL` for (0, 0, 0)
1947 - upper - The upper right corner, or `NULL` for (1, 1, 1)
1948 
1949   Output Parameter:
1950 . dm - The DM object
1951 
1952   Level: beginner
1953 
1954   Note:
1955   If you want to customize this mesh using options, you just need to
1956 .vb
1957   DMCreate(comm, &dm);
1958   DMSetType(dm, DMPLEX);
1959   DMSetFromOptions(dm);
1960 .ve
1961   and use the options on the `DMSetFromOptions()` page.
1962 
1963   The vertices are numbered is lexicographic order, and the dim edges exiting a vertex in the positive orthant are number consecutively,
1964 .vb
1965  18--0-19--2-20--4-18
1966   |     |     |     |
1967  13    15    17    13
1968   |     |     |     |
1969  24-12-25-14-26-16-24
1970   |     |     |     |
1971   7     9    11     7
1972   |     |     |     |
1973  21--6-22--8-23-10-21
1974   |     |     |     |
1975   1     3     5     1
1976   |     |     |     |
1977  18--0-19--2-20--4-18
1978 .ve
1979 
1980 .seealso: `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
1981 @*/
1982 PetscErrorCode DMPlexCreateHypercubicMesh(MPI_Comm comm, PetscInt dim, const PetscInt edges[], const PetscReal lower[], const PetscReal upper[], DM *dm)
1983 {
1984   PetscInt       *edg;
1985   PetscReal      *low, *upp;
1986   DMBoundaryType *bdt;
1987   PetscInt        d;
1988 
1989   PetscFunctionBegin;
1990   PetscCall(DMCreate(comm, dm));
1991   PetscCall(DMSetType(*dm, DMPLEX));
1992   PetscCall(PetscMalloc4(dim, &edg, dim, &low, dim, &upp, dim, &bdt));
1993   for (d = 0; d < dim; ++d) {
1994     edg[d] = edges ? edges[d] : 1;
1995     low[d] = lower ? lower[d] : 0.;
1996     upp[d] = upper ? upper[d] : 1.;
1997     bdt[d] = DM_BOUNDARY_PERIODIC;
1998   }
1999   PetscCall(DMPlexCreateHypercubicMesh_Internal(*dm, dim, low, upp, edg, bdt));
2000   PetscCall(PetscFree4(edg, low, upp, bdt));
2001   PetscFunctionReturn(PETSC_SUCCESS);
2002 }
2003 
2004 /*@
2005   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all `DM` options in the database.
2006 
2007   Logically Collective
2008 
2009   Input Parameters:
2010 + dm     - the `DM` context
2011 - prefix - the prefix to prepend to all option names
2012 
2013   Level: advanced
2014 
2015   Note:
2016   A hyphen (-) must NOT be given at the beginning of the prefix name.
2017   The first character of all runtime options is AUTOMATICALLY the hyphen.
2018 
2019 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `SNESSetFromOptions()`
2020 @*/
2021 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
2022 {
2023   DM_Plex *mesh = (DM_Plex *)dm->data;
2024 
2025   PetscFunctionBegin;
2026   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2027   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, prefix));
2028   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)mesh->partitioner, prefix));
2029   PetscFunctionReturn(PETSC_SUCCESS);
2030 }
2031 
2032 /* Remap geometry to cylinder
2033    TODO: This only works for a single refinement, then it is broken
2034 
2035      Interior square: Linear interpolation is correct
2036      The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
2037      such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
2038 
2039        phi     = arctan(y/x)
2040        d_close = sqrt(1/8 + 1/4 sin^2(phi))
2041        d_far   = sqrt(1/2 + sin^2(phi))
2042 
2043      so we remap them using
2044 
2045        x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
2046        y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
2047 
2048      If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
2049 */
2050 static void snapToCylinder(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
2051 {
2052   const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
2053   const PetscReal ds2 = 0.5 * dis;
2054 
2055   if ((PetscAbsScalar(u[0]) <= ds2) && (PetscAbsScalar(u[1]) <= ds2)) {
2056     f0[0] = u[0];
2057     f0[1] = u[1];
2058   } else {
2059     PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
2060 
2061     x    = PetscRealPart(u[0]);
2062     y    = PetscRealPart(u[1]);
2063     phi  = PetscAtan2Real(y, x);
2064     sinp = PetscSinReal(phi);
2065     cosp = PetscCosReal(phi);
2066     if ((PetscAbsReal(phi) > PETSC_PI / 4.0) && (PetscAbsReal(phi) < 3.0 * PETSC_PI / 4.0)) {
2067       dc = PetscAbsReal(ds2 / sinp);
2068       df = PetscAbsReal(dis / sinp);
2069       xc = ds2 * x / PetscAbsReal(y);
2070       yc = ds2 * PetscSignReal(y);
2071     } else {
2072       dc = PetscAbsReal(ds2 / cosp);
2073       df = PetscAbsReal(dis / cosp);
2074       xc = ds2 * PetscSignReal(x);
2075       yc = ds2 * y / PetscAbsReal(x);
2076     }
2077     f0[0] = xc + (u[0] - xc) * (1.0 - dc) / (df - dc);
2078     f0[1] = yc + (u[1] - yc) * (1.0 - dc) / (df - dc);
2079   }
2080   f0[2] = u[2];
2081 }
2082 
2083 static PetscErrorCode DMPlexCreateHexCylinderMesh_Internal(DM dm, DMBoundaryType periodicZ, PetscInt Nr)
2084 {
2085   const PetscInt dim = 3;
2086   PetscInt       numCells, numVertices;
2087   PetscMPIInt    rank;
2088 
2089   PetscFunctionBegin;
2090   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
2091   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2092   PetscCall(DMSetDimension(dm, dim));
2093   /* Create topology */
2094   {
2095     PetscInt cone[8], c;
2096 
2097     numCells    = rank == 0 ? 5 : 0;
2098     numVertices = rank == 0 ? 16 : 0;
2099     if (periodicZ == DM_BOUNDARY_PERIODIC) {
2100       numCells *= 3;
2101       numVertices = rank == 0 ? 24 : 0;
2102     }
2103     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
2104     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 8));
2105     PetscCall(DMSetUp(dm));
2106     if (rank == 0) {
2107       if (periodicZ == DM_BOUNDARY_PERIODIC) {
2108         cone[0] = 15;
2109         cone[1] = 18;
2110         cone[2] = 17;
2111         cone[3] = 16;
2112         cone[4] = 31;
2113         cone[5] = 32;
2114         cone[6] = 33;
2115         cone[7] = 34;
2116         PetscCall(DMPlexSetCone(dm, 0, cone));
2117         cone[0] = 16;
2118         cone[1] = 17;
2119         cone[2] = 24;
2120         cone[3] = 23;
2121         cone[4] = 32;
2122         cone[5] = 36;
2123         cone[6] = 37;
2124         cone[7] = 33; /* 22 25 26 21 */
2125         PetscCall(DMPlexSetCone(dm, 1, cone));
2126         cone[0] = 18;
2127         cone[1] = 27;
2128         cone[2] = 24;
2129         cone[3] = 17;
2130         cone[4] = 34;
2131         cone[5] = 33;
2132         cone[6] = 37;
2133         cone[7] = 38;
2134         PetscCall(DMPlexSetCone(dm, 2, cone));
2135         cone[0] = 29;
2136         cone[1] = 27;
2137         cone[2] = 18;
2138         cone[3] = 15;
2139         cone[4] = 35;
2140         cone[5] = 31;
2141         cone[6] = 34;
2142         cone[7] = 38;
2143         PetscCall(DMPlexSetCone(dm, 3, cone));
2144         cone[0] = 29;
2145         cone[1] = 15;
2146         cone[2] = 16;
2147         cone[3] = 23;
2148         cone[4] = 35;
2149         cone[5] = 36;
2150         cone[6] = 32;
2151         cone[7] = 31;
2152         PetscCall(DMPlexSetCone(dm, 4, cone));
2153 
2154         cone[0] = 31;
2155         cone[1] = 34;
2156         cone[2] = 33;
2157         cone[3] = 32;
2158         cone[4] = 19;
2159         cone[5] = 22;
2160         cone[6] = 21;
2161         cone[7] = 20;
2162         PetscCall(DMPlexSetCone(dm, 5, cone));
2163         cone[0] = 32;
2164         cone[1] = 33;
2165         cone[2] = 37;
2166         cone[3] = 36;
2167         cone[4] = 22;
2168         cone[5] = 25;
2169         cone[6] = 26;
2170         cone[7] = 21;
2171         PetscCall(DMPlexSetCone(dm, 6, cone));
2172         cone[0] = 34;
2173         cone[1] = 38;
2174         cone[2] = 37;
2175         cone[3] = 33;
2176         cone[4] = 20;
2177         cone[5] = 21;
2178         cone[6] = 26;
2179         cone[7] = 28;
2180         PetscCall(DMPlexSetCone(dm, 7, cone));
2181         cone[0] = 35;
2182         cone[1] = 38;
2183         cone[2] = 34;
2184         cone[3] = 31;
2185         cone[4] = 30;
2186         cone[5] = 19;
2187         cone[6] = 20;
2188         cone[7] = 28;
2189         PetscCall(DMPlexSetCone(dm, 8, cone));
2190         cone[0] = 35;
2191         cone[1] = 31;
2192         cone[2] = 32;
2193         cone[3] = 36;
2194         cone[4] = 30;
2195         cone[5] = 25;
2196         cone[6] = 22;
2197         cone[7] = 19;
2198         PetscCall(DMPlexSetCone(dm, 9, cone));
2199 
2200         cone[0] = 19;
2201         cone[1] = 20;
2202         cone[2] = 21;
2203         cone[3] = 22;
2204         cone[4] = 15;
2205         cone[5] = 16;
2206         cone[6] = 17;
2207         cone[7] = 18;
2208         PetscCall(DMPlexSetCone(dm, 10, cone));
2209         cone[0] = 22;
2210         cone[1] = 21;
2211         cone[2] = 26;
2212         cone[3] = 25;
2213         cone[4] = 16;
2214         cone[5] = 23;
2215         cone[6] = 24;
2216         cone[7] = 17;
2217         PetscCall(DMPlexSetCone(dm, 11, cone));
2218         cone[0] = 20;
2219         cone[1] = 28;
2220         cone[2] = 26;
2221         cone[3] = 21;
2222         cone[4] = 18;
2223         cone[5] = 17;
2224         cone[6] = 24;
2225         cone[7] = 27;
2226         PetscCall(DMPlexSetCone(dm, 12, cone));
2227         cone[0] = 30;
2228         cone[1] = 28;
2229         cone[2] = 20;
2230         cone[3] = 19;
2231         cone[4] = 29;
2232         cone[5] = 15;
2233         cone[6] = 18;
2234         cone[7] = 27;
2235         PetscCall(DMPlexSetCone(dm, 13, cone));
2236         cone[0] = 30;
2237         cone[1] = 19;
2238         cone[2] = 22;
2239         cone[3] = 25;
2240         cone[4] = 29;
2241         cone[5] = 23;
2242         cone[6] = 16;
2243         cone[7] = 15;
2244         PetscCall(DMPlexSetCone(dm, 14, cone));
2245       } else {
2246         cone[0] = 5;
2247         cone[1] = 8;
2248         cone[2] = 7;
2249         cone[3] = 6;
2250         cone[4] = 9;
2251         cone[5] = 12;
2252         cone[6] = 11;
2253         cone[7] = 10;
2254         PetscCall(DMPlexSetCone(dm, 0, cone));
2255         cone[0] = 6;
2256         cone[1] = 7;
2257         cone[2] = 14;
2258         cone[3] = 13;
2259         cone[4] = 12;
2260         cone[5] = 15;
2261         cone[6] = 16;
2262         cone[7] = 11;
2263         PetscCall(DMPlexSetCone(dm, 1, cone));
2264         cone[0] = 8;
2265         cone[1] = 17;
2266         cone[2] = 14;
2267         cone[3] = 7;
2268         cone[4] = 10;
2269         cone[5] = 11;
2270         cone[6] = 16;
2271         cone[7] = 18;
2272         PetscCall(DMPlexSetCone(dm, 2, cone));
2273         cone[0] = 19;
2274         cone[1] = 17;
2275         cone[2] = 8;
2276         cone[3] = 5;
2277         cone[4] = 20;
2278         cone[5] = 9;
2279         cone[6] = 10;
2280         cone[7] = 18;
2281         PetscCall(DMPlexSetCone(dm, 3, cone));
2282         cone[0] = 19;
2283         cone[1] = 5;
2284         cone[2] = 6;
2285         cone[3] = 13;
2286         cone[4] = 20;
2287         cone[5] = 15;
2288         cone[6] = 12;
2289         cone[7] = 9;
2290         PetscCall(DMPlexSetCone(dm, 4, cone));
2291       }
2292     }
2293     PetscCall(DMPlexSymmetrize(dm));
2294     PetscCall(DMPlexStratify(dm));
2295   }
2296   /* Create cube geometry */
2297   {
2298     Vec             coordinates;
2299     PetscSection    coordSection;
2300     PetscScalar    *coords;
2301     PetscInt        coordSize, v;
2302     const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
2303     const PetscReal ds2 = dis / 2.0;
2304 
2305     /* Build coordinates */
2306     PetscCall(DMGetCoordinateSection(dm, &coordSection));
2307     PetscCall(PetscSectionSetNumFields(coordSection, 1));
2308     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
2309     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
2310     for (v = numCells; v < numCells + numVertices; ++v) {
2311       PetscCall(PetscSectionSetDof(coordSection, v, dim));
2312       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
2313     }
2314     PetscCall(PetscSectionSetUp(coordSection));
2315     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2316     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2317     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2318     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2319     PetscCall(VecSetBlockSize(coordinates, dim));
2320     PetscCall(VecSetType(coordinates, VECSTANDARD));
2321     PetscCall(VecGetArray(coordinates, &coords));
2322     if (rank == 0) {
2323       coords[0 * dim + 0]  = -ds2;
2324       coords[0 * dim + 1]  = -ds2;
2325       coords[0 * dim + 2]  = 0.0;
2326       coords[1 * dim + 0]  = ds2;
2327       coords[1 * dim + 1]  = -ds2;
2328       coords[1 * dim + 2]  = 0.0;
2329       coords[2 * dim + 0]  = ds2;
2330       coords[2 * dim + 1]  = ds2;
2331       coords[2 * dim + 2]  = 0.0;
2332       coords[3 * dim + 0]  = -ds2;
2333       coords[3 * dim + 1]  = ds2;
2334       coords[3 * dim + 2]  = 0.0;
2335       coords[4 * dim + 0]  = -ds2;
2336       coords[4 * dim + 1]  = -ds2;
2337       coords[4 * dim + 2]  = 1.0;
2338       coords[5 * dim + 0]  = -ds2;
2339       coords[5 * dim + 1]  = ds2;
2340       coords[5 * dim + 2]  = 1.0;
2341       coords[6 * dim + 0]  = ds2;
2342       coords[6 * dim + 1]  = ds2;
2343       coords[6 * dim + 2]  = 1.0;
2344       coords[7 * dim + 0]  = ds2;
2345       coords[7 * dim + 1]  = -ds2;
2346       coords[7 * dim + 2]  = 1.0;
2347       coords[8 * dim + 0]  = dis;
2348       coords[8 * dim + 1]  = -dis;
2349       coords[8 * dim + 2]  = 0.0;
2350       coords[9 * dim + 0]  = dis;
2351       coords[9 * dim + 1]  = dis;
2352       coords[9 * dim + 2]  = 0.0;
2353       coords[10 * dim + 0] = dis;
2354       coords[10 * dim + 1] = -dis;
2355       coords[10 * dim + 2] = 1.0;
2356       coords[11 * dim + 0] = dis;
2357       coords[11 * dim + 1] = dis;
2358       coords[11 * dim + 2] = 1.0;
2359       coords[12 * dim + 0] = -dis;
2360       coords[12 * dim + 1] = dis;
2361       coords[12 * dim + 2] = 0.0;
2362       coords[13 * dim + 0] = -dis;
2363       coords[13 * dim + 1] = dis;
2364       coords[13 * dim + 2] = 1.0;
2365       coords[14 * dim + 0] = -dis;
2366       coords[14 * dim + 1] = -dis;
2367       coords[14 * dim + 2] = 0.0;
2368       coords[15 * dim + 0] = -dis;
2369       coords[15 * dim + 1] = -dis;
2370       coords[15 * dim + 2] = 1.0;
2371       if (periodicZ == DM_BOUNDARY_PERIODIC) {
2372         /* 15 31 19 */ coords[16 * dim + 0] = -ds2;
2373         coords[16 * dim + 1]                = -ds2;
2374         coords[16 * dim + 2]                = 0.5;
2375         /* 16 32 22 */ coords[17 * dim + 0] = ds2;
2376         coords[17 * dim + 1]                = -ds2;
2377         coords[17 * dim + 2]                = 0.5;
2378         /* 17 33 21 */ coords[18 * dim + 0] = ds2;
2379         coords[18 * dim + 1]                = ds2;
2380         coords[18 * dim + 2]                = 0.5;
2381         /* 18 34 20 */ coords[19 * dim + 0] = -ds2;
2382         coords[19 * dim + 1]                = ds2;
2383         coords[19 * dim + 2]                = 0.5;
2384         /* 29 35 30 */ coords[20 * dim + 0] = -dis;
2385         coords[20 * dim + 1]                = -dis;
2386         coords[20 * dim + 2]                = 0.5;
2387         /* 23 36 25 */ coords[21 * dim + 0] = dis;
2388         coords[21 * dim + 1]                = -dis;
2389         coords[21 * dim + 2]                = 0.5;
2390         /* 24 37 26 */ coords[22 * dim + 0] = dis;
2391         coords[22 * dim + 1]                = dis;
2392         coords[22 * dim + 2]                = 0.5;
2393         /* 27 38 28 */ coords[23 * dim + 0] = -dis;
2394         coords[23 * dim + 1]                = dis;
2395         coords[23 * dim + 2]                = 0.5;
2396       }
2397     }
2398     PetscCall(VecRestoreArray(coordinates, &coords));
2399     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2400     PetscCall(VecDestroy(&coordinates));
2401   }
2402   /* Create periodicity */
2403   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
2404     PetscReal L[3]       = {-1., -1., 0.};
2405     PetscReal maxCell[3] = {-1., -1., 0.};
2406     PetscReal lower[3]   = {0.0, 0.0, 0.0};
2407     PetscReal upper[3]   = {1.0, 1.0, 1.5};
2408     PetscInt  numZCells  = 3;
2409 
2410     L[2]       = upper[2] - lower[2];
2411     maxCell[2] = 1.1 * (L[2] / numZCells);
2412     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
2413   }
2414   {
2415     DM          cdm;
2416     PetscDS     cds;
2417     PetscScalar c[2] = {1.0, 1.0};
2418 
2419     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_TRUE, NULL));
2420     PetscCall(DMGetCoordinateDM(dm, &cdm));
2421     PetscCall(DMGetDS(cdm, &cds));
2422     PetscCall(PetscDSSetConstants(cds, 2, c));
2423   }
2424   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
2425 
2426   /* Wait for coordinate creation before doing in-place modification */
2427   PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2428 
2429   char        oldprefix[PETSC_MAX_PATH_LEN];
2430   const char *prefix;
2431 
2432   PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
2433   PetscCall(PetscStrncpy(oldprefix, prefix, PETSC_MAX_PATH_LEN));
2434   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, "petsc_cyl_ref_"));
2435   for (PetscInt r = 0; r < PetscMax(0, Nr); ++r) {
2436     DM rdm;
2437 
2438     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
2439     PetscCall(DMPlexReplace_Internal(dm, &rdm));
2440   }
2441   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, oldprefix));
2442   PetscCall(DMPlexRemapGeometry(dm, 0.0, snapToCylinder));
2443 
2444   DMLabel         bdlabel, edgelabel;
2445   IS              faceIS;
2446   const PetscInt *faces;
2447   PetscInt        Nf;
2448 
2449   PetscCall(DMCreateLabel(dm, "marker"));
2450   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
2451   PetscCall(DMCreateLabel(dm, "generatrix"));
2452   PetscCall(DMGetLabel(dm, "generatrix", &edgelabel));
2453   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
2454   // Remove faces on top and bottom
2455   PetscCall(DMLabelGetStratumIS(bdlabel, 1, &faceIS));
2456   if (faceIS) {
2457     PetscCall(ISGetLocalSize(faceIS, &Nf));
2458     PetscCall(ISGetIndices(faceIS, &faces));
2459     for (PetscInt f = 0; f < Nf; ++f) {
2460       PetscReal vol, normal[3];
2461 
2462       PetscCall(DMPlexComputeCellGeometryFVM(dm, faces[f], &vol, NULL, normal));
2463       if (PetscAbsReal(normal[2]) < PETSC_SMALL) PetscCall(DMLabelSetValue(edgelabel, faces[f], 1));
2464     }
2465     PetscCall(ISRestoreIndices(faceIS, &faces));
2466     PetscCall(ISDestroy(&faceIS));
2467   }
2468   PetscCall(DMPlexLabelComplete(dm, bdlabel));
2469   PetscCall(DMPlexLabelComplete(dm, edgelabel));
2470   PetscFunctionReturn(PETSC_SUCCESS);
2471 }
2472 
2473 /*@
2474   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
2475 
2476   Collective
2477 
2478   Input Parameters:
2479 + comm      - The communicator for the `DM` object
2480 . periodicZ - The boundary type for the Z direction
2481 - Nr        - The number of refinements to carry out
2482 
2483   Output Parameter:
2484 . dm - The `DM` object
2485 
2486   Level: beginner
2487 
2488   Note:
2489   Here is the output numbering looking from the bottom of the cylinder\:
2490 .vb
2491        17-----14
2492         |     |
2493         |  2  |
2494         |     |
2495  17-----8-----7-----14
2496   |     |     |     |
2497   |  3  |  0  |  1  |
2498   |     |     |     |
2499  19-----5-----6-----13
2500         |     |
2501         |  4  |
2502         |     |
2503        19-----13
2504 
2505  and up through the top
2506 
2507        18-----16
2508         |     |
2509         |  2  |
2510         |     |
2511  18----10----11-----16
2512   |     |     |     |
2513   |  3  |  0  |  1  |
2514   |     |     |     |
2515  20-----9----12-----15
2516         |     |
2517         |  4  |
2518         |     |
2519        20-----15
2520 .ve
2521 
2522 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2523 @*/
2524 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, DMBoundaryType periodicZ, PetscInt Nr, DM *dm)
2525 {
2526   PetscFunctionBegin;
2527   PetscAssertPointer(dm, 4);
2528   PetscCall(DMCreate(comm, dm));
2529   PetscCall(DMSetType(*dm, DMPLEX));
2530   PetscCall(DMPlexCreateHexCylinderMesh_Internal(*dm, periodicZ, Nr));
2531   PetscFunctionReturn(PETSC_SUCCESS);
2532 }
2533 
2534 static PetscErrorCode DMPlexCreateWedgeCylinderMesh_Internal(DM dm, PetscInt n, PetscBool interpolate)
2535 {
2536   const PetscInt dim = 3;
2537   PetscInt       numCells, numVertices, v;
2538   PetscMPIInt    rank;
2539 
2540   PetscFunctionBegin;
2541   PetscCheck(n >= 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %" PetscInt_FMT " cannot be negative", n);
2542   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
2543   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2544   PetscCall(DMSetDimension(dm, dim));
2545   /* Must create the celltype label here so that we do not automatically try to compute the types */
2546   PetscCall(DMCreateLabel(dm, "celltype"));
2547   /* Create topology */
2548   {
2549     PetscInt cone[6], c;
2550 
2551     numCells    = rank == 0 ? n : 0;
2552     numVertices = rank == 0 ? 2 * (n + 1) : 0;
2553     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
2554     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
2555     PetscCall(DMSetUp(dm));
2556     for (c = 0; c < numCells; c++) {
2557       cone[0] = c + n * 1;
2558       cone[1] = (c + 1) % n + n * 1;
2559       cone[2] = 0 + 3 * n;
2560       cone[3] = c + n * 2;
2561       cone[4] = (c + 1) % n + n * 2;
2562       cone[5] = 1 + 3 * n;
2563       PetscCall(DMPlexSetCone(dm, c, cone));
2564       PetscCall(DMPlexSetCellType(dm, c, DM_POLYTOPE_TRI_PRISM_TENSOR));
2565     }
2566     PetscCall(DMPlexSymmetrize(dm));
2567     PetscCall(DMPlexStratify(dm));
2568   }
2569   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetCellType(dm, v, DM_POLYTOPE_POINT));
2570   /* Create cylinder geometry */
2571   {
2572     Vec          coordinates;
2573     PetscSection coordSection;
2574     PetscScalar *coords;
2575     PetscInt     coordSize, c;
2576 
2577     /* Build coordinates */
2578     PetscCall(DMGetCoordinateSection(dm, &coordSection));
2579     PetscCall(PetscSectionSetNumFields(coordSection, 1));
2580     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
2581     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
2582     for (v = numCells; v < numCells + numVertices; ++v) {
2583       PetscCall(PetscSectionSetDof(coordSection, v, dim));
2584       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
2585     }
2586     PetscCall(PetscSectionSetUp(coordSection));
2587     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2588     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2589     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2590     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2591     PetscCall(VecSetBlockSize(coordinates, dim));
2592     PetscCall(VecSetType(coordinates, VECSTANDARD));
2593     PetscCall(VecGetArray(coordinates, &coords));
2594     for (c = 0; c < numCells; c++) {
2595       coords[(c + 0 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
2596       coords[(c + 0 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
2597       coords[(c + 0 * n) * dim + 2] = 1.0;
2598       coords[(c + 1 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
2599       coords[(c + 1 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
2600       coords[(c + 1 * n) * dim + 2] = 0.0;
2601     }
2602     if (rank == 0) {
2603       coords[(2 * n + 0) * dim + 0] = 0.0;
2604       coords[(2 * n + 0) * dim + 1] = 0.0;
2605       coords[(2 * n + 0) * dim + 2] = 1.0;
2606       coords[(2 * n + 1) * dim + 0] = 0.0;
2607       coords[(2 * n + 1) * dim + 1] = 0.0;
2608       coords[(2 * n + 1) * dim + 2] = 0.0;
2609     }
2610     PetscCall(VecRestoreArray(coordinates, &coords));
2611     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2612     PetscCall(VecDestroy(&coordinates));
2613   }
2614   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
2615   /* Interpolate */
2616   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2617   PetscFunctionReturn(PETSC_SUCCESS);
2618 }
2619 
2620 /*@
2621   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
2622 
2623   Collective
2624 
2625   Input Parameters:
2626 + comm        - The communicator for the `DM` object
2627 . n           - The number of wedges around the origin
2628 - interpolate - Create edges and faces
2629 
2630   Output Parameter:
2631 . dm - The `DM` object
2632 
2633   Level: beginner
2634 
2635 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2636 @*/
2637 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
2638 {
2639   PetscFunctionBegin;
2640   PetscAssertPointer(dm, 4);
2641   PetscCall(DMCreate(comm, dm));
2642   PetscCall(DMSetType(*dm, DMPLEX));
2643   PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(*dm, n, interpolate));
2644   PetscFunctionReturn(PETSC_SUCCESS);
2645 }
2646 
2647 static inline PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2648 {
2649   PetscReal prod = 0.0;
2650   PetscInt  i;
2651   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
2652   return PetscSqrtReal(prod);
2653 }
2654 
2655 static inline PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
2656 {
2657   PetscReal prod = 0.0;
2658   PetscInt  i;
2659   for (i = 0; i < dim; ++i) prod += x[i] * y[i];
2660   return prod;
2661 }
2662 
2663 /* The first constant is the sphere radius */
2664 static void snapToSphere(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
2665 {
2666   PetscReal r     = PetscRealPart(constants[0]);
2667   PetscReal norm2 = 0.0, fac;
2668   PetscInt  n     = uOff[1] - uOff[0], d;
2669 
2670   for (d = 0; d < n; ++d) norm2 += PetscSqr(PetscRealPart(u[d]));
2671   fac = r / PetscSqrtReal(norm2);
2672   for (d = 0; d < n; ++d) f0[d] = u[d] * fac;
2673 }
2674 
2675 static PetscErrorCode DMPlexCreateSphereMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, PetscReal R)
2676 {
2677   const PetscInt embedDim = dim + 1;
2678   PetscSection   coordSection;
2679   Vec            coordinates;
2680   PetscScalar   *coords;
2681   PetscReal     *coordsIn;
2682   PetscInt       numCells, numEdges, numVerts = 0, firstVertex = 0, v, firstEdge, coordSize, d, e;
2683   PetscMPIInt    rank;
2684 
2685   PetscFunctionBegin;
2686   PetscValidLogicalCollectiveBool(dm, simplex, 3);
2687   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
2688   PetscCall(DMSetDimension(dm, dim));
2689   PetscCall(DMSetCoordinateDim(dm, dim + 1));
2690   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2691   switch (dim) {
2692   case 1:
2693     numCells = 16;
2694     numVerts = numCells;
2695 
2696     // Build Topology
2697     PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2698     for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2699     PetscCall(DMSetUp(dm));
2700     for (PetscInt c = 0; c < numCells; ++c) {
2701       PetscInt cone[2];
2702 
2703       cone[0] = c + numCells;
2704       cone[1] = (c + 1) % numVerts + numCells;
2705       PetscCall(DMPlexSetCone(dm, c, cone));
2706     }
2707     PetscCall(DMPlexSymmetrize(dm));
2708     PetscCall(DMPlexStratify(dm));
2709     PetscCall(PetscMalloc1(numVerts * embedDim, &coordsIn));
2710     for (PetscInt v = 0; v < numVerts; ++v) {
2711       const PetscReal rad = 2. * PETSC_PI * v / numVerts;
2712 
2713       coordsIn[v * embedDim + 0] = PetscCosReal(rad);
2714       coordsIn[v * embedDim + 1] = PetscSinReal(rad);
2715     }
2716     break;
2717   case 2:
2718     if (simplex) {
2719       const PetscReal radius    = PetscSqrtReal(1 + PETSC_PHI * PETSC_PHI) / (1.0 + PETSC_PHI);
2720       const PetscReal edgeLen   = 2.0 / (1.0 + PETSC_PHI) * (R / radius);
2721       const PetscInt  degree    = 5;
2722       PetscReal       vertex[3] = {0.0, 1.0 / (1.0 + PETSC_PHI), PETSC_PHI / (1.0 + PETSC_PHI)};
2723       PetscInt        s[3]      = {1, 1, 1};
2724       PetscInt        cone[3];
2725       PetscInt       *graph;
2726 
2727       vertex[0] *= R / radius;
2728       vertex[1] *= R / radius;
2729       vertex[2] *= R / radius;
2730       numCells    = rank == 0 ? 20 : 0;
2731       numVerts    = rank == 0 ? 12 : 0;
2732       firstVertex = numCells;
2733       /* Use icosahedron, which for a R-sphere has coordinates which are all cyclic permutations of
2734 
2735            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
2736 
2737          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
2738          length is then given by 2/(1+\phi) = 2 * 0.38197 = 0.76393.
2739       */
2740       /* Construct vertices */
2741       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2742       if (rank == 0) {
2743         for (PetscInt p = 0, i = 0; p < embedDim; ++p) {
2744           for (s[1] = -1; s[1] < 2; s[1] += 2) {
2745             for (s[2] = -1; s[2] < 2; s[2] += 2) {
2746               for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertex[(d + p) % embedDim];
2747               ++i;
2748             }
2749           }
2750         }
2751       }
2752       /* Construct graph */
2753       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
2754       for (PetscInt i = 0; i < numVerts; ++i) {
2755         PetscInt k = 0;
2756         for (PetscInt j = 0; j < numVerts; ++j) {
2757           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
2758             graph[i * numVerts + j] = 1;
2759             ++k;
2760           }
2761         }
2762         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid icosahedron, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
2763       }
2764       /* Build Topology */
2765       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
2766       for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
2767       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2768       /* Cells */
2769       for (PetscInt i = 0, c = 0; i < numVerts; ++i) {
2770         for (PetscInt j = 0; j < i; ++j) {
2771           for (PetscInt k = 0; k < j; ++k) {
2772             if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i]) {
2773               cone[0] = firstVertex + i;
2774               cone[1] = firstVertex + j;
2775               cone[2] = firstVertex + k;
2776               /* Check orientation */
2777               {
2778                 const PetscInt epsilon[3][3][3] = {
2779                   {{0, 0, 0},  {0, 0, 1},  {0, -1, 0}},
2780                   {{0, 0, -1}, {0, 0, 0},  {1, 0, 0} },
2781                   {{0, 1, 0},  {-1, 0, 0}, {0, 0, 0} }
2782                 };
2783                 PetscReal normal[3];
2784                 PetscInt  e, f;
2785 
2786                 for (d = 0; d < embedDim; ++d) {
2787                   normal[d] = 0.0;
2788                   for (e = 0; e < embedDim; ++e) {
2789                     for (f = 0; f < embedDim; ++f) normal[d] += epsilon[d][e][f] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]);
2790                   }
2791                 }
2792                 if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
2793                   PetscInt tmp = cone[1];
2794                   cone[1]      = cone[2];
2795                   cone[2]      = tmp;
2796                 }
2797               }
2798               PetscCall(DMPlexSetCone(dm, c++, cone));
2799             }
2800           }
2801         }
2802       }
2803       PetscCall(DMPlexSymmetrize(dm));
2804       PetscCall(DMPlexStratify(dm));
2805       PetscCall(PetscFree(graph));
2806     } else {
2807       /*
2808         12-21--13
2809          |     |
2810         25  4  24
2811          |     |
2812   12-25--9-16--8-24--13
2813    |     |     |     |
2814   23  5 17  0 15  3  22
2815    |     |     |     |
2816   10-20--6-14--7-19--11
2817          |     |
2818         20  1  19
2819          |     |
2820         10-18--11
2821          |     |
2822         23  2  22
2823          |     |
2824         12-21--13
2825        */
2826       PetscInt cone[4], ornt[4];
2827 
2828       numCells    = rank == 0 ? 6 : 0;
2829       numEdges    = rank == 0 ? 12 : 0;
2830       numVerts    = rank == 0 ? 8 : 0;
2831       firstVertex = numCells;
2832       firstEdge   = numCells + numVerts;
2833       /* Build Topology */
2834       PetscCall(DMPlexSetChart(dm, 0, numCells + numEdges + numVerts));
2835       for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 4));
2836       for (e = firstEdge; e < firstEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
2837       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
2838       if (rank == 0) {
2839         /* Cell 0 */
2840         cone[0] = 14;
2841         cone[1] = 15;
2842         cone[2] = 16;
2843         cone[3] = 17;
2844         PetscCall(DMPlexSetCone(dm, 0, cone));
2845         ornt[0] = 0;
2846         ornt[1] = 0;
2847         ornt[2] = 0;
2848         ornt[3] = 0;
2849         PetscCall(DMPlexSetConeOrientation(dm, 0, ornt));
2850         /* Cell 1 */
2851         cone[0] = 18;
2852         cone[1] = 19;
2853         cone[2] = 14;
2854         cone[3] = 20;
2855         PetscCall(DMPlexSetCone(dm, 1, cone));
2856         ornt[0] = 0;
2857         ornt[1] = 0;
2858         ornt[2] = -1;
2859         ornt[3] = 0;
2860         PetscCall(DMPlexSetConeOrientation(dm, 1, ornt));
2861         /* Cell 2 */
2862         cone[0] = 21;
2863         cone[1] = 22;
2864         cone[2] = 18;
2865         cone[3] = 23;
2866         PetscCall(DMPlexSetCone(dm, 2, cone));
2867         ornt[0] = 0;
2868         ornt[1] = 0;
2869         ornt[2] = -1;
2870         ornt[3] = 0;
2871         PetscCall(DMPlexSetConeOrientation(dm, 2, ornt));
2872         /* Cell 3 */
2873         cone[0] = 19;
2874         cone[1] = 22;
2875         cone[2] = 24;
2876         cone[3] = 15;
2877         PetscCall(DMPlexSetCone(dm, 3, cone));
2878         ornt[0] = -1;
2879         ornt[1] = -1;
2880         ornt[2] = 0;
2881         ornt[3] = -1;
2882         PetscCall(DMPlexSetConeOrientation(dm, 3, ornt));
2883         /* Cell 4 */
2884         cone[0] = 16;
2885         cone[1] = 24;
2886         cone[2] = 21;
2887         cone[3] = 25;
2888         PetscCall(DMPlexSetCone(dm, 4, cone));
2889         ornt[0] = -1;
2890         ornt[1] = -1;
2891         ornt[2] = -1;
2892         ornt[3] = 0;
2893         PetscCall(DMPlexSetConeOrientation(dm, 4, ornt));
2894         /* Cell 5 */
2895         cone[0] = 20;
2896         cone[1] = 17;
2897         cone[2] = 25;
2898         cone[3] = 23;
2899         PetscCall(DMPlexSetCone(dm, 5, cone));
2900         ornt[0] = -1;
2901         ornt[1] = -1;
2902         ornt[2] = -1;
2903         ornt[3] = -1;
2904         PetscCall(DMPlexSetConeOrientation(dm, 5, ornt));
2905         /* Edges */
2906         cone[0] = 6;
2907         cone[1] = 7;
2908         PetscCall(DMPlexSetCone(dm, 14, cone));
2909         cone[0] = 7;
2910         cone[1] = 8;
2911         PetscCall(DMPlexSetCone(dm, 15, cone));
2912         cone[0] = 8;
2913         cone[1] = 9;
2914         PetscCall(DMPlexSetCone(dm, 16, cone));
2915         cone[0] = 9;
2916         cone[1] = 6;
2917         PetscCall(DMPlexSetCone(dm, 17, cone));
2918         cone[0] = 10;
2919         cone[1] = 11;
2920         PetscCall(DMPlexSetCone(dm, 18, cone));
2921         cone[0] = 11;
2922         cone[1] = 7;
2923         PetscCall(DMPlexSetCone(dm, 19, cone));
2924         cone[0] = 6;
2925         cone[1] = 10;
2926         PetscCall(DMPlexSetCone(dm, 20, cone));
2927         cone[0] = 12;
2928         cone[1] = 13;
2929         PetscCall(DMPlexSetCone(dm, 21, cone));
2930         cone[0] = 13;
2931         cone[1] = 11;
2932         PetscCall(DMPlexSetCone(dm, 22, cone));
2933         cone[0] = 10;
2934         cone[1] = 12;
2935         PetscCall(DMPlexSetCone(dm, 23, cone));
2936         cone[0] = 13;
2937         cone[1] = 8;
2938         PetscCall(DMPlexSetCone(dm, 24, cone));
2939         cone[0] = 12;
2940         cone[1] = 9;
2941         PetscCall(DMPlexSetCone(dm, 25, cone));
2942       }
2943       PetscCall(DMPlexSymmetrize(dm));
2944       PetscCall(DMPlexStratify(dm));
2945       /* Build coordinates */
2946       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
2947       if (rank == 0) {
2948         coordsIn[0 * embedDim + 0] = -R;
2949         coordsIn[0 * embedDim + 1] = R;
2950         coordsIn[0 * embedDim + 2] = -R;
2951         coordsIn[1 * embedDim + 0] = R;
2952         coordsIn[1 * embedDim + 1] = R;
2953         coordsIn[1 * embedDim + 2] = -R;
2954         coordsIn[2 * embedDim + 0] = R;
2955         coordsIn[2 * embedDim + 1] = -R;
2956         coordsIn[2 * embedDim + 2] = -R;
2957         coordsIn[3 * embedDim + 0] = -R;
2958         coordsIn[3 * embedDim + 1] = -R;
2959         coordsIn[3 * embedDim + 2] = -R;
2960         coordsIn[4 * embedDim + 0] = -R;
2961         coordsIn[4 * embedDim + 1] = R;
2962         coordsIn[4 * embedDim + 2] = R;
2963         coordsIn[5 * embedDim + 0] = R;
2964         coordsIn[5 * embedDim + 1] = R;
2965         coordsIn[5 * embedDim + 2] = R;
2966         coordsIn[6 * embedDim + 0] = -R;
2967         coordsIn[6 * embedDim + 1] = -R;
2968         coordsIn[6 * embedDim + 2] = R;
2969         coordsIn[7 * embedDim + 0] = R;
2970         coordsIn[7 * embedDim + 1] = -R;
2971         coordsIn[7 * embedDim + 2] = R;
2972       }
2973     }
2974     break;
2975   case 3:
2976     if (simplex) {
2977       const PetscReal edgeLen         = 1.0 / PETSC_PHI;
2978       PetscReal       vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
2979       PetscReal       vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
2980       PetscReal       vertexC[4]      = {0.5, 0.5 * PETSC_PHI, 0.5 / PETSC_PHI, 0.0};
2981       const PetscInt  degree          = 12;
2982       PetscInt        s[4]            = {1, 1, 1};
2983       PetscInt        evenPerm[12][4] = {
2984         {0, 1, 2, 3},
2985         {0, 2, 3, 1},
2986         {0, 3, 1, 2},
2987         {1, 0, 3, 2},
2988         {1, 2, 0, 3},
2989         {1, 3, 2, 0},
2990         {2, 0, 1, 3},
2991         {2, 1, 3, 0},
2992         {2, 3, 0, 1},
2993         {3, 0, 2, 1},
2994         {3, 1, 0, 2},
2995         {3, 2, 1, 0}
2996       };
2997       PetscInt  cone[4];
2998       PetscInt *graph, p, i, j, k, l;
2999 
3000       vertexA[0] *= R;
3001       vertexA[1] *= R;
3002       vertexA[2] *= R;
3003       vertexA[3] *= R;
3004       vertexB[0] *= R;
3005       vertexB[1] *= R;
3006       vertexB[2] *= R;
3007       vertexB[3] *= R;
3008       vertexC[0] *= R;
3009       vertexC[1] *= R;
3010       vertexC[2] *= R;
3011       vertexC[3] *= R;
3012       numCells    = rank == 0 ? 600 : 0;
3013       numVerts    = rank == 0 ? 120 : 0;
3014       firstVertex = numCells;
3015       /* Use the 600-cell, which for a unit sphere has coordinates which are
3016 
3017            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
3018                (\pm 1,    0,       0,      0)  all cyclic permutations   8
3019            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
3020 
3021          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
3022          length is then given by 1/\phi = 0.61803.
3023 
3024          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
3025          http://mathworld.wolfram.com/600-Cell.html
3026       */
3027       /* Construct vertices */
3028       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
3029       i = 0;
3030       if (rank == 0) {
3031         for (s[0] = -1; s[0] < 2; s[0] += 2) {
3032           for (s[1] = -1; s[1] < 2; s[1] += 2) {
3033             for (s[2] = -1; s[2] < 2; s[2] += 2) {
3034               for (s[3] = -1; s[3] < 2; s[3] += 2) {
3035                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[d] * vertexA[d];
3036                 ++i;
3037               }
3038             }
3039           }
3040         }
3041         for (p = 0; p < embedDim; ++p) {
3042           s[1] = s[2] = s[3] = 1;
3043           for (s[0] = -1; s[0] < 2; s[0] += 2) {
3044             for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertexB[(d + p) % embedDim];
3045             ++i;
3046           }
3047         }
3048         for (p = 0; p < 12; ++p) {
3049           s[3] = 1;
3050           for (s[0] = -1; s[0] < 2; s[0] += 2) {
3051             for (s[1] = -1; s[1] < 2; s[1] += 2) {
3052               for (s[2] = -1; s[2] < 2; s[2] += 2) {
3053                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[evenPerm[p][d]] * vertexC[evenPerm[p][d]];
3054                 ++i;
3055               }
3056             }
3057           }
3058         }
3059       }
3060       PetscCheck(i == numVerts, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertices %" PetscInt_FMT " != %" PetscInt_FMT, i, numVerts);
3061       /* Construct graph */
3062       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
3063       for (i = 0; i < numVerts; ++i) {
3064         for (j = 0, k = 0; j < numVerts; ++j) {
3065           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
3066             graph[i * numVerts + j] = 1;
3067             ++k;
3068           }
3069         }
3070         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
3071       }
3072       /* Build Topology */
3073       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
3074       for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
3075       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
3076       /* Cells */
3077       if (rank == 0) {
3078         for (PetscInt i = 0, c = 0; i < numVerts; ++i) {
3079           for (j = 0; j < i; ++j) {
3080             for (k = 0; k < j; ++k) {
3081               for (l = 0; l < k; ++l) {
3082                 if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i] && graph[l * numVerts + i] && graph[l * numVerts + j] && graph[l * numVerts + k]) {
3083                   cone[0] = firstVertex + i;
3084                   cone[1] = firstVertex + j;
3085                   cone[2] = firstVertex + k;
3086                   cone[3] = firstVertex + l;
3087                   /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
3088                   {
3089                     const PetscInt epsilon[4][4][4][4] = {
3090                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, -1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 0, 0}, {0, 1, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 1, 0}, {0, -1, 0, 0}, {0, 0, 0, 0}}},
3091 
3092                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}}, {{0, 0, -1, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}}},
3093 
3094                       {{{0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, 0, 0}, {0, -1, 0, 0}}, {{0, 0, 0, -1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 1, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}},
3095 
3096                       {{{0, 0, 0, 0}, {0, 0, -1, 0}, {0, 1, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 1, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}}, {{0, -1, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}} }
3097                     };
3098                     PetscReal normal[4];
3099                     PetscInt  e, f, g;
3100 
3101                     for (d = 0; d < embedDim; ++d) {
3102                       normal[d] = 0.0;
3103                       for (e = 0; e < embedDim; ++e) {
3104                         for (f = 0; f < embedDim; ++f) {
3105                           for (g = 0; g < embedDim; ++g) {
3106                             normal[d] += epsilon[d][e][f][g] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]) * (coordsIn[l * embedDim + f] - coordsIn[i * embedDim + f]);
3107                           }
3108                         }
3109                       }
3110                     }
3111                     if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
3112                       PetscInt tmp = cone[1];
3113                       cone[1]      = cone[2];
3114                       cone[2]      = tmp;
3115                     }
3116                   }
3117                   PetscCall(DMPlexSetCone(dm, c++, cone));
3118                 }
3119               }
3120             }
3121           }
3122         }
3123       }
3124       PetscCall(DMPlexSymmetrize(dm));
3125       PetscCall(DMPlexStratify(dm));
3126       PetscCall(PetscFree(graph));
3127     }
3128     break;
3129   default:
3130     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unsupported dimension for sphere: %" PetscInt_FMT, dim);
3131   }
3132   /* Create coordinates */
3133   PetscCall(DMGetCoordinateSection(dm, &coordSection));
3134   PetscCall(PetscSectionSetNumFields(coordSection, 1));
3135   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, embedDim));
3136   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVerts));
3137   for (v = firstVertex; v < firstVertex + numVerts; ++v) {
3138     PetscCall(PetscSectionSetDof(coordSection, v, embedDim));
3139     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, embedDim));
3140   }
3141   PetscCall(PetscSectionSetUp(coordSection));
3142   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
3143   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
3144   PetscCall(VecSetBlockSize(coordinates, embedDim));
3145   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
3146   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
3147   PetscCall(VecSetType(coordinates, VECSTANDARD));
3148   PetscCall(VecGetArray(coordinates, &coords));
3149   for (v = 0; v < numVerts; ++v)
3150     for (d = 0; d < embedDim; ++d) coords[v * embedDim + d] = coordsIn[v * embedDim + d];
3151   PetscCall(VecRestoreArray(coordinates, &coords));
3152   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
3153   PetscCall(VecDestroy(&coordinates));
3154   PetscCall(PetscFree(coordsIn));
3155   {
3156     DM          cdm;
3157     PetscDS     cds;
3158     PetscScalar c = R;
3159 
3160     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_TRUE, snapToSphere));
3161     PetscCall(DMGetCoordinateDM(dm, &cdm));
3162     PetscCall(DMGetDS(cdm, &cds));
3163     PetscCall(PetscDSSetConstants(cds, 1, &c));
3164   }
3165   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
3166   /* Wait for coordinate creation before doing in-place modification */
3167   if (simplex) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
3168   PetscFunctionReturn(PETSC_SUCCESS);
3169 }
3170 
3171 typedef void (*TPSEvaluateFunc)(const PetscReal[], PetscReal *, PetscReal[], PetscReal (*)[3]);
3172 
3173 /*
3174  The Schwarz P implicit surface is
3175 
3176      f(x) = cos(x0) + cos(x1) + cos(x2) = 0
3177 */
3178 static void TPSEvaluate_SchwarzP(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
3179 {
3180   PetscReal c[3] = {PetscCosReal(y[0] * PETSC_PI), PetscCosReal(y[1] * PETSC_PI), PetscCosReal(y[2] * PETSC_PI)};
3181   PetscReal g[3] = {-PetscSinReal(y[0] * PETSC_PI), -PetscSinReal(y[1] * PETSC_PI), -PetscSinReal(y[2] * PETSC_PI)};
3182   f[0]           = c[0] + c[1] + c[2];
3183   for (PetscInt i = 0; i < 3; i++) {
3184     grad[i] = PETSC_PI * g[i];
3185     for (PetscInt j = 0; j < 3; j++) hess[i][j] = (i == j) ? -PetscSqr(PETSC_PI) * c[i] : 0.;
3186   }
3187 }
3188 
3189 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
3190 static PetscErrorCode TPSExtrudeNormalFunc_SchwarzP(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
3191 {
3192   for (PetscInt i = 0; i < 3; i++) u[i] = -PETSC_PI * PetscSinReal(x[i] * PETSC_PI);
3193   return PETSC_SUCCESS;
3194 }
3195 
3196 /*
3197  The Gyroid implicit surface is
3198 
3199  f(x,y,z) = sin(pi * x) * cos (pi * (y + 1/2))  + sin(pi * (y + 1/2)) * cos(pi * (z + 1/4)) + sin(pi * (z + 1/4)) * cos(pi * x)
3200 
3201 */
3202 static void TPSEvaluate_Gyroid(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
3203 {
3204   PetscReal s[3] = {PetscSinReal(PETSC_PI * y[0]), PetscSinReal(PETSC_PI * (y[1] + .5)), PetscSinReal(PETSC_PI * (y[2] + .25))};
3205   PetscReal c[3] = {PetscCosReal(PETSC_PI * y[0]), PetscCosReal(PETSC_PI * (y[1] + .5)), PetscCosReal(PETSC_PI * (y[2] + .25))};
3206   f[0]           = s[0] * c[1] + s[1] * c[2] + s[2] * c[0];
3207   grad[0]        = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
3208   grad[1]        = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
3209   grad[2]        = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
3210   hess[0][0]     = -PetscSqr(PETSC_PI) * (s[0] * c[1] + s[2] * c[0]);
3211   hess[0][1]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
3212   hess[0][2]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
3213   hess[1][0]     = -PetscSqr(PETSC_PI) * (s[1] * c[2] + s[0] * c[1]);
3214   hess[1][1]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
3215   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
3216   hess[2][0]     = -PetscSqr(PETSC_PI) * (s[2] * c[0] + s[1] * c[2]);
3217   hess[2][1]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
3218   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
3219 }
3220 
3221 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
3222 static PetscErrorCode TPSExtrudeNormalFunc_Gyroid(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
3223 {
3224   PetscReal s[3] = {PetscSinReal(PETSC_PI * x[0]), PetscSinReal(PETSC_PI * (x[1] + .5)), PetscSinReal(PETSC_PI * (x[2] + .25))};
3225   PetscReal c[3] = {PetscCosReal(PETSC_PI * x[0]), PetscCosReal(PETSC_PI * (x[1] + .5)), PetscCosReal(PETSC_PI * (x[2] + .25))};
3226   u[0]           = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
3227   u[1]           = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
3228   u[2]           = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
3229   return PETSC_SUCCESS;
3230 }
3231 
3232 /*
3233    We wish to solve
3234 
3235          min_y || y - x ||^2  subject to f(y) = 0
3236 
3237    Let g(y) = grad(f).  The minimization problem is equivalent to asking to satisfy
3238    f(y) = 0 and (y-x) is parallel to g(y).  We do this by using Householder QR to obtain a basis for the
3239    tangent space and ask for both components in the tangent space to be zero.
3240 
3241    Take g to be a column vector and compute the "full QR" factorization Q R = g,
3242    where Q = I - 2 n n^T is a symmetric orthogonal matrix.
3243    The first column of Q is parallel to g so the remaining two columns span the null space.
3244    Let Qn = Q[:,1:] be those remaining columns.  Then Qn Qn^T is an orthogonal projector into the tangent space.
3245    Since Q is symmetric, this is equivalent to multiplying by Q and taking the last two entries.
3246    In total, we have a system of 3 equations in 3 unknowns:
3247 
3248      f(y) = 0                       1 equation
3249      Qn^T (y - x) = 0               2 equations
3250 
3251    Here, we compute the residual and Jacobian of this system.
3252 */
3253 static void TPSNearestPointResJac(TPSEvaluateFunc feval, const PetscScalar x[], const PetscScalar y[], PetscScalar res[], PetscScalar J[])
3254 {
3255   PetscReal yreal[3] = {PetscRealPart(y[0]), PetscRealPart(y[1]), PetscRealPart(y[2])};
3256   PetscReal d[3]     = {PetscRealPart(y[0] - x[0]), PetscRealPart(y[1] - x[1]), PetscRealPart(y[2] - x[2])};
3257   PetscReal f, grad[3], n[3], norm, norm_y[3], nd, nd_y[3], sign;
3258   PetscReal n_y[3][3] = {
3259     {0, 0, 0},
3260     {0, 0, 0},
3261     {0, 0, 0}
3262   };
3263 
3264   feval(yreal, &f, grad, n_y);
3265 
3266   for (PetscInt i = 0; i < 3; i++) n[i] = grad[i];
3267   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
3268   for (PetscInt i = 0; i < 3; i++) norm_y[i] = 1. / norm * n[i] * n_y[i][i];
3269 
3270   // Define the Householder reflector
3271   sign = n[0] >= 0 ? 1. : -1.;
3272   n[0] += norm * sign;
3273   for (PetscInt i = 0; i < 3; i++) n_y[0][i] += norm_y[i] * sign;
3274 
3275   norm      = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
3276   norm_y[0] = 1. / norm * (n[0] * n_y[0][0]);
3277   norm_y[1] = 1. / norm * (n[0] * n_y[0][1] + n[1] * n_y[1][1]);
3278   norm_y[2] = 1. / norm * (n[0] * n_y[0][2] + n[2] * n_y[2][2]);
3279 
3280   for (PetscInt i = 0; i < 3; i++) {
3281     n[i] /= norm;
3282     for (PetscInt j = 0; j < 3; j++) {
3283       // note that n[i] is n_old[i]/norm when executing the code below
3284       n_y[i][j] = n_y[i][j] / norm - n[i] / norm * norm_y[j];
3285     }
3286   }
3287 
3288   nd = n[0] * d[0] + n[1] * d[1] + n[2] * d[2];
3289   for (PetscInt i = 0; i < 3; i++) nd_y[i] = n[i] + n_y[0][i] * d[0] + n_y[1][i] * d[1] + n_y[2][i] * d[2];
3290 
3291   res[0] = f;
3292   res[1] = d[1] - 2 * n[1] * nd;
3293   res[2] = d[2] - 2 * n[2] * nd;
3294   // J[j][i] is J_{ij} (column major)
3295   for (PetscInt j = 0; j < 3; j++) {
3296     J[0 + j * 3] = grad[j];
3297     J[1 + j * 3] = (j == 1) * 1. - 2 * (n_y[1][j] * nd + n[1] * nd_y[j]);
3298     J[2 + j * 3] = (j == 2) * 1. - 2 * (n_y[2][j] * nd + n[2] * nd_y[j]);
3299   }
3300 }
3301 
3302 /*
3303    Project x to the nearest point on the implicit surface using Newton's method.
3304 */
3305 static PetscErrorCode TPSNearestPoint(TPSEvaluateFunc feval, PetscScalar x[])
3306 {
3307   PetscScalar y[3] = {x[0], x[1], x[2]}; // Initial guess
3308 
3309   PetscFunctionBegin;
3310   for (PetscInt iter = 0; iter < 10; iter++) {
3311     PetscScalar res[3], J[9];
3312     PetscReal   resnorm;
3313     TPSNearestPointResJac(feval, x, y, res, J);
3314     resnorm = PetscSqrtReal(PetscSqr(PetscRealPart(res[0])) + PetscSqr(PetscRealPart(res[1])) + PetscSqr(PetscRealPart(res[2])));
3315     if (0) { // Turn on this monitor if you need to confirm quadratic convergence
3316       PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%" PetscInt_FMT "] res [%g %g %g]\n", iter, (double)PetscRealPart(res[0]), (double)PetscRealPart(res[1]), (double)PetscRealPart(res[2])));
3317     }
3318     if (resnorm < PETSC_SMALL) break;
3319 
3320     // Take the Newton step
3321     PetscCall(PetscKernel_A_gets_inverse_A_3(J, 0., PETSC_FALSE, NULL));
3322     PetscKernel_v_gets_v_minus_A_times_w_3(y, J, res);
3323   }
3324   for (PetscInt i = 0; i < 3; i++) x[i] = y[i];
3325   PetscFunctionReturn(PETSC_SUCCESS);
3326 }
3327 
3328 const char *const DMPlexTPSTypes[] = {"SCHWARZ_P", "GYROID", "DMPlexTPSType", "DMPLEX_TPS_", NULL};
3329 
3330 static PetscErrorCode DMPlexCreateTPSMesh_Internal(DM dm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness)
3331 {
3332   PetscMPIInt rank;
3333   PetscInt    topoDim = 2, spaceDim = 3, numFaces = 0, numVertices = 0, numEdges = 0;
3334   PetscInt(*edges)[2] = NULL, *edgeSets = NULL;
3335   PetscInt           *cells_flat = NULL;
3336   PetscReal          *vtxCoords  = NULL;
3337   TPSEvaluateFunc     evalFunc   = NULL;
3338   PetscSimplePointFn *normalFunc = NULL;
3339   DMLabel             label;
3340 
3341   PetscFunctionBegin;
3342   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
3343   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3344   PetscCheck((layers != 0) ^ (thickness == 0.), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Layers %" PetscInt_FMT " must be nonzero iff thickness %g is nonzero", layers, (double)thickness);
3345   switch (tpstype) {
3346   case DMPLEX_TPS_SCHWARZ_P:
3347     PetscCheck(!periodic || (periodic[0] == DM_BOUNDARY_NONE && periodic[1] == DM_BOUNDARY_NONE && periodic[2] == DM_BOUNDARY_NONE), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Schwarz P does not support periodic meshes");
3348     if (rank == 0) {
3349       PetscInt(*cells)[6][4][4] = NULL; // [junction, junction-face, cell, conn]
3350       PetscInt  Njunctions = 0, Ncuts = 0, Npipes[3], vcount;
3351       PetscReal L = 1;
3352 
3353       Npipes[0]   = (extent[0] + 1) * extent[1] * extent[2];
3354       Npipes[1]   = extent[0] * (extent[1] + 1) * extent[2];
3355       Npipes[2]   = extent[0] * extent[1] * (extent[2] + 1);
3356       Njunctions  = extent[0] * extent[1] * extent[2];
3357       Ncuts       = 2 * (extent[0] * extent[1] + extent[1] * extent[2] + extent[2] * extent[0]);
3358       numVertices = 4 * (Npipes[0] + Npipes[1] + Npipes[2]) + 8 * Njunctions;
3359       PetscCall(PetscMalloc1(3 * numVertices, &vtxCoords));
3360       PetscCall(PetscMalloc1(Njunctions, &cells));
3361       PetscCall(PetscMalloc1(Ncuts * 4, &edges));
3362       PetscCall(PetscMalloc1(Ncuts * 4, &edgeSets));
3363       // x-normal pipes
3364       vcount = 0;
3365       for (PetscInt i = 0; i < extent[0] + 1; i++) {
3366         for (PetscInt j = 0; j < extent[1]; j++) {
3367           for (PetscInt k = 0; k < extent[2]; k++) {
3368             for (PetscInt l = 0; l < 4; l++) {
3369               vtxCoords[vcount++] = (2 * i - 1) * L;
3370               vtxCoords[vcount++] = 2 * j * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3371               vtxCoords[vcount++] = 2 * k * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3372             }
3373           }
3374         }
3375       }
3376       // y-normal pipes
3377       for (PetscInt i = 0; i < extent[0]; i++) {
3378         for (PetscInt j = 0; j < extent[1] + 1; j++) {
3379           for (PetscInt k = 0; k < extent[2]; k++) {
3380             for (PetscInt l = 0; l < 4; l++) {
3381               vtxCoords[vcount++] = 2 * i * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3382               vtxCoords[vcount++] = (2 * j - 1) * L;
3383               vtxCoords[vcount++] = 2 * k * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3384             }
3385           }
3386         }
3387       }
3388       // z-normal pipes
3389       for (PetscInt i = 0; i < extent[0]; i++) {
3390         for (PetscInt j = 0; j < extent[1]; j++) {
3391           for (PetscInt k = 0; k < extent[2] + 1; k++) {
3392             for (PetscInt l = 0; l < 4; l++) {
3393               vtxCoords[vcount++] = 2 * i * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3394               vtxCoords[vcount++] = 2 * j * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3395               vtxCoords[vcount++] = (2 * k - 1) * L;
3396             }
3397           }
3398         }
3399       }
3400       // junctions
3401       for (PetscInt i = 0; i < extent[0]; i++) {
3402         for (PetscInt j = 0; j < extent[1]; j++) {
3403           for (PetscInt k = 0; k < extent[2]; k++) {
3404             const PetscInt J = (i * extent[1] + j) * extent[2] + k, Jvoff = (Npipes[0] + Npipes[1] + Npipes[2]) * 4 + J * 8;
3405             PetscCheck(vcount / 3 == Jvoff, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected vertex count");
3406             for (PetscInt ii = 0; ii < 2; ii++) {
3407               for (PetscInt jj = 0; jj < 2; jj++) {
3408                 for (PetscInt kk = 0; kk < 2; kk++) {
3409                   double Ls           = (1 - sqrt(2) / 4) * L;
3410                   vtxCoords[vcount++] = 2 * i * L + (2 * ii - 1) * Ls;
3411                   vtxCoords[vcount++] = 2 * j * L + (2 * jj - 1) * Ls;
3412                   vtxCoords[vcount++] = 2 * k * L + (2 * kk - 1) * Ls;
3413                 }
3414               }
3415             }
3416             const PetscInt jfaces[3][2][4] = {
3417               {{3, 1, 0, 2}, {7, 5, 4, 6}}, // x-aligned
3418               {{5, 4, 0, 1}, {7, 6, 2, 3}}, // y-aligned
3419               {{6, 2, 0, 4}, {7, 3, 1, 5}}  // z-aligned
3420             };
3421             const PetscInt pipe_lo[3] = {// vertex numbers of pipes
3422                                          ((i * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + Npipes[0] + Npipes[1]) * 4};
3423             const PetscInt pipe_hi[3] = {// vertex numbers of pipes
3424                                          (((i + 1) * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j + 1) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + 1 + Npipes[0] + Npipes[1]) * 4};
3425             for (PetscInt dir = 0; dir < 3; dir++) { // x,y,z
3426               const PetscInt ijk[3] = {i, j, k};
3427               for (PetscInt l = 0; l < 4; l++) { // rotations
3428                 cells[J][dir * 2 + 0][l][0] = pipe_lo[dir] + l;
3429                 cells[J][dir * 2 + 0][l][1] = Jvoff + jfaces[dir][0][l];
3430                 cells[J][dir * 2 + 0][l][2] = Jvoff + jfaces[dir][0][(l - 1 + 4) % 4];
3431                 cells[J][dir * 2 + 0][l][3] = pipe_lo[dir] + (l - 1 + 4) % 4;
3432                 cells[J][dir * 2 + 1][l][0] = Jvoff + jfaces[dir][1][l];
3433                 cells[J][dir * 2 + 1][l][1] = pipe_hi[dir] + l;
3434                 cells[J][dir * 2 + 1][l][2] = pipe_hi[dir] + (l - 1 + 4) % 4;
3435                 cells[J][dir * 2 + 1][l][3] = Jvoff + jfaces[dir][1][(l - 1 + 4) % 4];
3436                 if (ijk[dir] == 0) {
3437                   edges[numEdges][0] = pipe_lo[dir] + l;
3438                   edges[numEdges][1] = pipe_lo[dir] + (l + 1) % 4;
3439                   edgeSets[numEdges] = dir * 2 + 1;
3440                   numEdges++;
3441                 }
3442                 if (ijk[dir] + 1 == extent[dir]) {
3443                   edges[numEdges][0] = pipe_hi[dir] + l;
3444                   edges[numEdges][1] = pipe_hi[dir] + (l + 1) % 4;
3445                   edgeSets[numEdges] = dir * 2 + 2;
3446                   numEdges++;
3447                 }
3448               }
3449             }
3450           }
3451         }
3452       }
3453       PetscCheck(numEdges == Ncuts * 4, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Edge count %" PetscInt_FMT " incompatible with number of cuts %" PetscInt_FMT, numEdges, Ncuts);
3454       numFaces   = 24 * Njunctions;
3455       cells_flat = cells[0][0][0];
3456     }
3457     evalFunc   = TPSEvaluate_SchwarzP;
3458     normalFunc = TPSExtrudeNormalFunc_SchwarzP;
3459     break;
3460   case DMPLEX_TPS_GYROID:
3461     if (rank == 0) {
3462       // This is a coarse mesh approximation of the gyroid shifted to being the zero of the level set
3463       //
3464       //     sin(pi*x)*cos(pi*(y+1/2)) + sin(pi*(y+1/2))*cos(pi*(z+1/4)) + sin(pi*(z+1/4))*cos(x)
3465       //
3466       // on the cell [0,2]^3.
3467       //
3468       // Think about dividing that cell into four columns, and focus on the column [0,1]x[0,1]x[0,2].
3469       // If you looked at the gyroid in that column at different slices of z you would see that it kind of spins
3470       // like a boomerang:
3471       //
3472       //     z = 0          z = 1/4        z = 1/2        z = 3/4     //
3473       //     -----          -------        -------        -------     //
3474       //                                                              //
3475       //     +       +      +       +      +       +      +   \   +   //
3476       //      \                                   /            \      //
3477       //       \            `-_   _-'            /              }     //
3478       //        *-_            `-'            _-'              /      //
3479       //     +     `-+      +       +      +-'     +      +   /   +   //
3480       //                                                              //
3481       //                                                              //
3482       //     z = 1          z = 5/4        z = 3/2        z = 7/4     //
3483       //     -----          -------        -------        -------     //
3484       //                                                              //
3485       //     +-_     +      +       +      +     _-+      +   /   +   //
3486       //        `-_            _-_            _-`            /        //
3487       //           \        _-'   `-_        /              {         //
3488       //            \                       /                \        //
3489       //     +       +      +       +      +       +      +   \   +   //
3490       //
3491       //
3492       // This course mesh approximates each of these slices by two line segments,
3493       // and then connects the segments in consecutive layers with quadrilateral faces.
3494       // All of the end points of the segments are multiples of 1/4 except for the
3495       // point * in the picture for z = 0 above and the similar points in other layers.
3496       // That point is at (gamma, gamma, 0), where gamma is calculated below.
3497       //
3498       // The column  [1,2]x[1,2]x[0,2] looks the same as this column;
3499       // The columns [1,2]x[0,1]x[0,2] and [0,1]x[1,2]x[0,2] are mirror images.
3500       //
3501       // As for how this method turned into the names given to the vertices:
3502       // that was not systematic, it was just the way it worked out in my handwritten notes.
3503 
3504       PetscInt facesPerBlock = 64;
3505       PetscInt vertsPerBlock = 56;
3506       PetscInt extentPlus[3];
3507       PetscInt numBlocks, numBlocksPlus;
3508       const PetscInt A = 0, B = 1, C = 2, D = 3, E = 4, F = 5, G = 6, H = 7, II = 8, J = 9, K = 10, L = 11, M = 12, N = 13, O = 14, P = 15, Q = 16, R = 17, S = 18, T = 19, U = 20, V = 21, W = 22, X = 23, Y = 24, Z = 25, Ap = 26, Bp = 27, Cp = 28, Dp = 29, Ep = 30, Fp = 31, Gp = 32, Hp = 33, Ip = 34, Jp = 35, Kp = 36, Lp = 37, Mp = 38, Np = 39, Op = 40, Pp = 41, Qp = 42, Rp = 43, Sp = 44, Tp = 45, Up = 46, Vp = 47, Wp = 48, Xp = 49, Yp = 50, Zp = 51, Aq = 52, Bq = 53, Cq = 54, Dq = 55;
3509       const PetscInt pattern[64][4] = {
3510         /* face to vertex within the coarse discretization of a single gyroid block */
3511         /* layer 0 */
3512         {A,           C,           K,           G          },
3513         {C,           B,           II,          K          },
3514         {D,           A,           H,           L          },
3515         {B + 56 * 1,  D,           L,           J          },
3516         {E,           B + 56 * 1,  J,           N          },
3517         {A + 56 * 2,  E,           N,           H + 56 * 2 },
3518         {F,           A + 56 * 2,  G + 56 * 2,  M          },
3519         {B,           F,           M,           II         },
3520         /* layer 1 */
3521         {G,           K,           Q,           O          },
3522         {K,           II,          P,           Q          },
3523         {L,           H,           O + 56 * 1,  R          },
3524         {J,           L,           R,           P          },
3525         {N,           J,           P,           S          },
3526         {H + 56 * 2,  N,           S,           O + 56 * 3 },
3527         {M,           G + 56 * 2,  O + 56 * 2,  T          },
3528         {II,          M,           T,           P          },
3529         /* layer 2 */
3530         {O,           Q,           Y,           U          },
3531         {Q,           P,           W,           Y          },
3532         {R,           O + 56 * 1,  U + 56 * 1,  Ap         },
3533         {P,           R,           Ap,          W          },
3534         {S,           P,           X,           Bp         },
3535         {O + 56 * 3,  S,           Bp,          V + 56 * 1 },
3536         {T,           O + 56 * 2,  V,           Z          },
3537         {P,           T,           Z,           X          },
3538         /* layer 3 */
3539         {U,           Y,           Ep,          Dp         },
3540         {Y,           W,           Cp,          Ep         },
3541         {Ap,          U + 56 * 1,  Dp + 56 * 1, Gp         },
3542         {W,           Ap,          Gp,          Cp         },
3543         {Bp,          X,           Cp + 56 * 2, Fp         },
3544         {V + 56 * 1,  Bp,          Fp,          Dp + 56 * 1},
3545         {Z,           V,           Dp,          Hp         },
3546         {X,           Z,           Hp,          Cp + 56 * 2},
3547         /* layer 4 */
3548         {Dp,          Ep,          Mp,          Kp         },
3549         {Ep,          Cp,          Ip,          Mp         },
3550         {Gp,          Dp + 56 * 1, Lp,          Np         },
3551         {Cp,          Gp,          Np,          Jp         },
3552         {Fp,          Cp + 56 * 2, Jp + 56 * 2, Pp         },
3553         {Dp + 56 * 1, Fp,          Pp,          Lp         },
3554         {Hp,          Dp,          Kp,          Op         },
3555         {Cp + 56 * 2, Hp,          Op,          Ip + 56 * 2},
3556         /* layer 5 */
3557         {Kp,          Mp,          Sp,          Rp         },
3558         {Mp,          Ip,          Qp,          Sp         },
3559         {Np,          Lp,          Rp,          Tp         },
3560         {Jp,          Np,          Tp,          Qp + 56 * 1},
3561         {Pp,          Jp + 56 * 2, Qp + 56 * 3, Up         },
3562         {Lp,          Pp,          Up,          Rp         },
3563         {Op,          Kp,          Rp,          Vp         },
3564         {Ip + 56 * 2, Op,          Vp,          Qp + 56 * 2},
3565         /* layer 6 */
3566         {Rp,          Sp,          Aq,          Yp         },
3567         {Sp,          Qp,          Wp,          Aq         },
3568         {Tp,          Rp,          Yp,          Cq         },
3569         {Qp + 56 * 1, Tp,          Cq,          Wp + 56 * 1},
3570         {Up,          Qp + 56 * 3, Xp + 56 * 1, Dq         },
3571         {Rp,          Up,          Dq,          Zp         },
3572         {Vp,          Rp,          Zp,          Bq         },
3573         {Qp + 56 * 2, Vp,          Bq,          Xp         },
3574         /* layer 7 (the top is the periodic image of the bottom of layer 0) */
3575         {Yp,          Aq,          C + 56 * 4,  A + 56 * 4 },
3576         {Aq,          Wp,          B + 56 * 4,  C + 56 * 4 },
3577         {Cq,          Yp,          A + 56 * 4,  D + 56 * 4 },
3578         {Wp + 56 * 1, Cq,          D + 56 * 4,  B + 56 * 5 },
3579         {Dq,          Xp + 56 * 1, B + 56 * 5,  E + 56 * 4 },
3580         {Zp,          Dq,          E + 56 * 4,  A + 56 * 6 },
3581         {Bq,          Zp,          A + 56 * 6,  F + 56 * 4 },
3582         {Xp,          Bq,          F + 56 * 4,  B + 56 * 4 }
3583       };
3584       const PetscReal gamma                = PetscAcosReal((PetscSqrtReal(3.) - 1.) / PetscSqrtReal(2.)) / PETSC_PI;
3585       const PetscReal patternCoords[56][3] = {
3586         {1.,        0.,        0.  }, /* A  */
3587         {0.,        1.,        0.  }, /* B  */
3588         {gamma,     gamma,     0.  }, /* C  */
3589         {1 + gamma, 1 - gamma, 0.  }, /* D  */
3590         {2 - gamma, 2 - gamma, 0.  }, /* E  */
3591         {1 - gamma, 1 + gamma, 0.  }, /* F  */
3592 
3593         {.5,        0,         .25 }, /* G  */
3594         {1.5,       0.,        .25 }, /* H  */
3595         {.5,        1.,        .25 }, /* II */
3596         {1.5,       1.,        .25 }, /* J  */
3597         {.25,       .5,        .25 }, /* K  */
3598         {1.25,      .5,        .25 }, /* L  */
3599         {.75,       1.5,       .25 }, /* M  */
3600         {1.75,      1.5,       .25 }, /* N  */
3601 
3602         {0.,        0.,        .5  }, /* O  */
3603         {1.,        1.,        .5  }, /* P  */
3604         {gamma,     1 - gamma, .5  }, /* Q  */
3605         {1 + gamma, gamma,     .5  }, /* R  */
3606         {2 - gamma, 1 + gamma, .5  }, /* S  */
3607         {1 - gamma, 2 - gamma, .5  }, /* T  */
3608 
3609         {0.,        .5,        .75 }, /* U  */
3610         {0.,        1.5,       .75 }, /* V  */
3611         {1.,        .5,        .75 }, /* W  */
3612         {1.,        1.5,       .75 }, /* X  */
3613         {.5,        .75,       .75 }, /* Y  */
3614         {.5,        1.75,      .75 }, /* Z  */
3615         {1.5,       .25,       .75 }, /* Ap */
3616         {1.5,       1.25,      .75 }, /* Bp */
3617 
3618         {1.,        0.,        1.  }, /* Cp */
3619         {0.,        1.,        1.  }, /* Dp */
3620         {1 - gamma, 1 - gamma, 1.  }, /* Ep */
3621         {1 + gamma, 1 + gamma, 1.  }, /* Fp */
3622         {2 - gamma, gamma,     1.  }, /* Gp */
3623         {gamma,     2 - gamma, 1.  }, /* Hp */
3624 
3625         {.5,        0.,        1.25}, /* Ip */
3626         {1.5,       0.,        1.25}, /* Jp */
3627         {.5,        1.,        1.25}, /* Kp */
3628         {1.5,       1.,        1.25}, /* Lp */
3629         {.75,       .5,        1.25}, /* Mp */
3630         {1.75,      .5,        1.25}, /* Np */
3631         {.25,       1.5,       1.25}, /* Op */
3632         {1.25,      1.5,       1.25}, /* Pp */
3633 
3634         {0.,        0.,        1.5 }, /* Qp */
3635         {1.,        1.,        1.5 }, /* Rp */
3636         {1 - gamma, gamma,     1.5 }, /* Sp */
3637         {2 - gamma, 1 - gamma, 1.5 }, /* Tp */
3638         {1 + gamma, 2 - gamma, 1.5 }, /* Up */
3639         {gamma,     1 + gamma, 1.5 }, /* Vp */
3640 
3641         {0.,        .5,        1.75}, /* Wp */
3642         {0.,        1.5,       1.75}, /* Xp */
3643         {1.,        .5,        1.75}, /* Yp */
3644         {1.,        1.5,       1.75}, /* Zp */
3645         {.5,        .25,       1.75}, /* Aq */
3646         {.5,        1.25,      1.75}, /* Bq */
3647         {1.5,       .75,       1.75}, /* Cq */
3648         {1.5,       1.75,      1.75}, /* Dq */
3649       };
3650       PetscInt(*cells)[64][4] = NULL;
3651       PetscBool *seen;
3652       PetscInt  *vertToTrueVert;
3653       PetscInt   count;
3654 
3655       for (PetscInt i = 0; i < 3; i++) extentPlus[i] = extent[i] + 1;
3656       numBlocks = 1;
3657       for (PetscInt i = 0; i < 3; i++) numBlocks *= extent[i];
3658       numBlocksPlus = 1;
3659       for (PetscInt i = 0; i < 3; i++) numBlocksPlus *= extentPlus[i];
3660       numFaces = numBlocks * facesPerBlock;
3661       PetscCall(PetscMalloc1(numBlocks, &cells));
3662       PetscCall(PetscCalloc1(numBlocksPlus * vertsPerBlock, &seen));
3663       for (PetscInt k = 0; k < extent[2]; k++) {
3664         for (PetscInt j = 0; j < extent[1]; j++) {
3665           for (PetscInt i = 0; i < extent[0]; i++) {
3666             for (PetscInt f = 0; f < facesPerBlock; f++) {
3667               for (PetscInt v = 0; v < 4; v++) {
3668                 PetscInt vertRaw     = pattern[f][v];
3669                 PetscInt blockidx    = vertRaw / 56;
3670                 PetscInt patternvert = vertRaw % 56;
3671                 PetscInt xplus       = (blockidx & 1);
3672                 PetscInt yplus       = (blockidx & 2) >> 1;
3673                 PetscInt zplus       = (blockidx & 4) >> 2;
3674                 PetscInt zcoord      = (periodic && periodic[2] == DM_BOUNDARY_PERIODIC) ? ((k + zplus) % extent[2]) : (k + zplus);
3675                 PetscInt ycoord      = (periodic && periodic[1] == DM_BOUNDARY_PERIODIC) ? ((j + yplus) % extent[1]) : (j + yplus);
3676                 PetscInt xcoord      = (periodic && periodic[0] == DM_BOUNDARY_PERIODIC) ? ((i + xplus) % extent[0]) : (i + xplus);
3677                 PetscInt vert        = ((zcoord * extentPlus[1] + ycoord) * extentPlus[0] + xcoord) * 56 + patternvert;
3678 
3679                 cells[(k * extent[1] + j) * extent[0] + i][f][v] = vert;
3680                 seen[vert]                                       = PETSC_TRUE;
3681               }
3682             }
3683           }
3684         }
3685       }
3686       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++)
3687         if (seen[i]) numVertices++;
3688       count = 0;
3689       PetscCall(PetscMalloc1(numBlocksPlus * vertsPerBlock, &vertToTrueVert));
3690       PetscCall(PetscMalloc1(numVertices * 3, &vtxCoords));
3691       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) vertToTrueVert[i] = -1;
3692       for (PetscInt k = 0; k < extentPlus[2]; k++) {
3693         for (PetscInt j = 0; j < extentPlus[1]; j++) {
3694           for (PetscInt i = 0; i < extentPlus[0]; i++) {
3695             for (PetscInt v = 0; v < vertsPerBlock; v++) {
3696               PetscInt vIdx = ((k * extentPlus[1] + j) * extentPlus[0] + i) * vertsPerBlock + v;
3697 
3698               if (seen[vIdx]) {
3699                 PetscInt thisVert;
3700 
3701                 vertToTrueVert[vIdx] = thisVert = count++;
3702 
3703                 for (PetscInt d = 0; d < 3; d++) vtxCoords[3 * thisVert + d] = patternCoords[v][d];
3704                 vtxCoords[3 * thisVert + 0] += i * 2;
3705                 vtxCoords[3 * thisVert + 1] += j * 2;
3706                 vtxCoords[3 * thisVert + 2] += k * 2;
3707               }
3708             }
3709           }
3710         }
3711       }
3712       for (PetscInt i = 0; i < numBlocks; i++) {
3713         for (PetscInt f = 0; f < facesPerBlock; f++) {
3714           for (PetscInt v = 0; v < 4; v++) cells[i][f][v] = vertToTrueVert[cells[i][f][v]];
3715         }
3716       }
3717       PetscCall(PetscFree(vertToTrueVert));
3718       PetscCall(PetscFree(seen));
3719       cells_flat = cells[0][0];
3720       numEdges   = 0;
3721       for (PetscInt i = 0; i < numFaces; i++) {
3722         for (PetscInt e = 0; e < 4; e++) {
3723           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3724           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3725 
3726           for (PetscInt d = 0; d < 3; d++) {
3727             if (!periodic || periodic[0] != DM_BOUNDARY_PERIODIC) {
3728               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) numEdges++;
3729               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) numEdges++;
3730             }
3731           }
3732         }
3733       }
3734       PetscCall(PetscMalloc1(numEdges, &edges));
3735       PetscCall(PetscMalloc1(numEdges, &edgeSets));
3736       for (PetscInt edge = 0, i = 0; i < numFaces; i++) {
3737         for (PetscInt e = 0; e < 4; e++) {
3738           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
3739           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
3740 
3741           for (PetscInt d = 0; d < 3; d++) {
3742             if (!periodic || periodic[d] != DM_BOUNDARY_PERIODIC) {
3743               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) {
3744                 edges[edge][0]   = ev[0];
3745                 edges[edge][1]   = ev[1];
3746                 edgeSets[edge++] = 2 * d;
3747               }
3748               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) {
3749                 edges[edge][0]   = ev[0];
3750                 edges[edge][1]   = ev[1];
3751                 edgeSets[edge++] = 2 * d + 1;
3752               }
3753             }
3754           }
3755         }
3756       }
3757     }
3758     evalFunc   = TPSEvaluate_Gyroid;
3759     normalFunc = TPSExtrudeNormalFunc_Gyroid;
3760     break;
3761   }
3762 
3763   PetscCall(DMSetDimension(dm, topoDim));
3764   if (rank == 0) PetscCall(DMPlexBuildFromCellList(dm, numFaces, numVertices, 4, cells_flat));
3765   else PetscCall(DMPlexBuildFromCellList(dm, 0, 0, 0, NULL));
3766   PetscCall(PetscFree(cells_flat));
3767   {
3768     DM idm;
3769     PetscCall(DMPlexInterpolate(dm, &idm));
3770     PetscCall(DMPlexReplace_Internal(dm, &idm));
3771   }
3772   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, vtxCoords));
3773   else PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, NULL));
3774   PetscCall(PetscFree(vtxCoords));
3775 
3776   PetscCall(DMCreateLabel(dm, "Face Sets"));
3777   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3778   for (PetscInt e = 0; e < numEdges; e++) {
3779     PetscInt        njoin;
3780     const PetscInt *join, verts[] = {numFaces + edges[e][0], numFaces + edges[e][1]};
3781     PetscCall(DMPlexGetJoin(dm, 2, verts, &njoin, &join));
3782     PetscCheck(njoin == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected unique join of vertices %" PetscInt_FMT " and %" PetscInt_FMT, edges[e][0], edges[e][1]);
3783     PetscCall(DMLabelSetValue(label, join[0], edgeSets[e]));
3784     PetscCall(DMPlexRestoreJoin(dm, 2, verts, &njoin, &join));
3785   }
3786   PetscCall(PetscFree(edges));
3787   PetscCall(PetscFree(edgeSets));
3788   if (tps_distribute) {
3789     DM               pdm = NULL;
3790     PetscPartitioner part;
3791 
3792     PetscCall(DMPlexGetPartitioner(dm, &part));
3793     PetscCall(PetscPartitionerSetFromOptions(part));
3794     PetscCall(DMPlexDistribute(dm, 0, NULL, &pdm));
3795     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
3796     // Do not auto-distribute again
3797     PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
3798   }
3799 
3800   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
3801   for (PetscInt refine = 0; refine < refinements; refine++) {
3802     PetscInt     m;
3803     DM           dmf;
3804     Vec          X;
3805     PetscScalar *x;
3806     PetscCall(DMRefine(dm, MPI_COMM_NULL, &dmf));
3807     PetscCall(DMPlexReplace_Internal(dm, &dmf));
3808 
3809     PetscCall(DMGetCoordinatesLocal(dm, &X));
3810     PetscCall(VecGetLocalSize(X, &m));
3811     PetscCall(VecGetArray(X, &x));
3812     for (PetscInt i = 0; i < m; i += 3) PetscCall(TPSNearestPoint(evalFunc, &x[i]));
3813     PetscCall(VecRestoreArray(X, &x));
3814   }
3815 
3816   // Face Sets has already been propagated to new vertices during refinement; this propagates to the initial vertices.
3817   PetscCall(DMGetLabel(dm, "Face Sets", &label));
3818   PetscCall(DMPlexLabelComplete(dm, label));
3819 
3820   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
3821 
3822   if (thickness > 0) {
3823     DM              edm, cdm, ecdm;
3824     DMPlexTransform tr;
3825     const char     *prefix;
3826     PetscOptions    options;
3827     // Code from DMPlexExtrude
3828     PetscCall(DMPlexTransformCreate(PetscObjectComm((PetscObject)dm), &tr));
3829     PetscCall(DMPlexTransformSetDM(tr, dm));
3830     PetscCall(DMPlexTransformSetType(tr, DMPLEXEXTRUDE));
3831     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
3832     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)tr, prefix));
3833     PetscCall(PetscObjectGetOptions((PetscObject)dm, &options));
3834     PetscCall(PetscObjectSetOptions((PetscObject)tr, options));
3835     PetscCall(DMPlexTransformExtrudeSetLayers(tr, layers));
3836     PetscCall(DMPlexTransformExtrudeSetThickness(tr, thickness));
3837     PetscCall(DMPlexTransformExtrudeSetTensor(tr, PETSC_FALSE));
3838     PetscCall(DMPlexTransformExtrudeSetSymmetric(tr, PETSC_TRUE));
3839     PetscCall(DMPlexTransformExtrudeSetNormalFunction(tr, normalFunc));
3840     PetscCall(DMPlexTransformSetFromOptions(tr));
3841     PetscCall(PetscObjectSetOptions((PetscObject)tr, NULL));
3842     PetscCall(DMPlexTransformSetUp(tr));
3843     PetscCall(PetscObjectViewFromOptions((PetscObject)tr, NULL, "-dm_plex_tps_transform_view"));
3844     PetscCall(DMPlexTransformApply(tr, dm, &edm));
3845     PetscCall(DMCopyDisc(dm, edm));
3846     PetscCall(DMGetCoordinateDM(dm, &cdm));
3847     PetscCall(DMGetCoordinateDM(edm, &ecdm));
3848     PetscCall(DMCopyDisc(cdm, ecdm));
3849     PetscCall(DMPlexTransformCreateDiscLabels(tr, edm));
3850     PetscCall(DMPlexTransformDestroy(&tr));
3851     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, edm));
3852     PetscCall(DMPlexReplace_Internal(dm, &edm));
3853   }
3854   PetscFunctionReturn(PETSC_SUCCESS);
3855 }
3856 
3857 /*@
3858   DMPlexCreateTPSMesh - Create a distributed, interpolated mesh of a triply-periodic surface
3859 
3860   Collective
3861 
3862   Input Parameters:
3863 + comm           - The communicator for the `DM` object
3864 . tpstype        - Type of triply-periodic surface
3865 . extent         - Array of length 3 containing number of periods in each direction
3866 . periodic       - array of length 3 with periodicity, or `NULL` for non-periodic
3867 . tps_distribute - Distribute 2D manifold mesh prior to refinement and extrusion (more scalable)
3868 . refinements    - Number of factor-of-2 refinements of 2D manifold mesh
3869 . layers         - Number of cell layers extruded in normal direction
3870 - thickness      - Thickness in normal direction
3871 
3872   Output Parameter:
3873 . dm - The `DM` object
3874 
3875   Level: beginner
3876 
3877   Notes:
3878   This meshes the surface of the Schwarz P or Gyroid surfaces.  Schwarz P is the simplest member of the triply-periodic minimal surfaces.
3879   <https://en.wikipedia.org/wiki/Schwarz_minimal_surface#Schwarz_P_(%22Primitive%22)> and can be cut with "clean" boundaries.
3880   The Gyroid <https://en.wikipedia.org/wiki/Gyroid> is another triply-periodic minimal surface with applications in additive manufacturing; it is much more difficult to "cut" since there are no planes of symmetry.
3881   Our implementation creates a very coarse mesh of the surface and refines (by 4-way splitting) as many times as requested.
3882   On each refinement, all vertices are projected to their nearest point on the surface.
3883   This projection could readily be extended to related surfaces.
3884 
3885   See {cite}`maskery2018insights`
3886 
3887   The face (edge) sets for the Schwarz P surface are numbered $1(-x), 2(+x), 3(-y), 4(+y), 5(-z), 6(+z)$.
3888   When the mesh is refined, "Face Sets" contain the new vertices (created during refinement).
3889   Use `DMPlexLabelComplete()` to propagate to coarse-level vertices.
3890 
3891   Developer Notes:
3892   The Gyroid mesh does not currently mark boundary sets.
3893 
3894 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMSetType()`, `DMCreate()`
3895 @*/
3896 PetscErrorCode DMPlexCreateTPSMesh(MPI_Comm comm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness, DM *dm)
3897 {
3898   PetscFunctionBegin;
3899   PetscCall(DMCreate(comm, dm));
3900   PetscCall(DMSetType(*dm, DMPLEX));
3901   PetscCall(DMPlexCreateTPSMesh_Internal(*dm, tpstype, extent, periodic, tps_distribute, refinements, layers, thickness));
3902   PetscFunctionReturn(PETSC_SUCCESS);
3903 }
3904 
3905 /*@
3906   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
3907 
3908   Collective
3909 
3910   Input Parameters:
3911 + comm    - The communicator for the `DM` object
3912 . dim     - The dimension
3913 . simplex - Use simplices, or tensor product cells
3914 - R       - The radius
3915 
3916   Output Parameter:
3917 . dm - The `DM` object
3918 
3919   Level: beginner
3920 
3921 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBallMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3922 @*/
3923 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscReal R, DM *dm)
3924 {
3925   PetscFunctionBegin;
3926   PetscAssertPointer(dm, 5);
3927   PetscCall(DMCreate(comm, dm));
3928   PetscCall(DMSetType(*dm, DMPLEX));
3929   PetscCall(DMPlexCreateSphereMesh_Internal(*dm, dim, simplex, R));
3930   PetscFunctionReturn(PETSC_SUCCESS);
3931 }
3932 
3933 static PetscErrorCode DMPlexCreateBallMesh_Internal(DM dm, PetscInt dim, PetscReal R)
3934 {
3935   DM          sdm, vol;
3936   DMLabel     bdlabel;
3937   const char *prefix;
3938 
3939   PetscFunctionBegin;
3940   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
3941   PetscCall(DMSetType(sdm, DMPLEX));
3942   PetscCall(DMGetOptionsPrefix(dm, &prefix));
3943   PetscCall(DMSetOptionsPrefix(sdm, prefix));
3944   PetscCall(DMAppendOptionsPrefix(sdm, "bd_"));
3945   PetscCall(DMPlexDistributeSetDefault(sdm, PETSC_FALSE));
3946   PetscCall(DMPlexCreateSphereMesh_Internal(sdm, dim - 1, PETSC_TRUE, R));
3947   PetscCall(DMSetFromOptions(sdm));
3948   PetscCall(DMViewFromOptions(sdm, NULL, "-dm_view"));
3949   PetscCall(DMPlexGenerate(sdm, NULL, PETSC_TRUE, &vol));
3950   PetscCall(DMDestroy(&sdm));
3951   PetscCall(DMPlexReplace_Internal(dm, &vol));
3952   PetscCall(DMCreateLabel(dm, "marker"));
3953   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
3954   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
3955   PetscCall(DMPlexLabelComplete(dm, bdlabel));
3956   PetscFunctionReturn(PETSC_SUCCESS);
3957 }
3958 
3959 /*@
3960   DMPlexCreateBallMesh - Creates a simplex mesh on the d-dimensional ball, B^d.
3961 
3962   Collective
3963 
3964   Input Parameters:
3965 + comm - The communicator for the `DM` object
3966 . dim  - The dimension
3967 - R    - The radius
3968 
3969   Output Parameter:
3970 . dm - The `DM` object
3971 
3972   Options Database Key:
3973 . bd_dm_refine - This will refine the surface mesh preserving the sphere geometry
3974 
3975   Level: beginner
3976 
3977 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3978 @*/
3979 PetscErrorCode DMPlexCreateBallMesh(MPI_Comm comm, PetscInt dim, PetscReal R, DM *dm)
3980 {
3981   PetscFunctionBegin;
3982   PetscCall(DMCreate(comm, dm));
3983   PetscCall(DMSetType(*dm, DMPLEX));
3984   PetscCall(DMPlexCreateBallMesh_Internal(*dm, dim, R));
3985   PetscFunctionReturn(PETSC_SUCCESS);
3986 }
3987 
3988 static PetscErrorCode DMPlexCreateReferenceCell_Internal(DM rdm, DMPolytopeType ct)
3989 {
3990   PetscFunctionBegin;
3991   switch (ct) {
3992   case DM_POLYTOPE_POINT: {
3993     PetscInt    numPoints[1]        = {1};
3994     PetscInt    coneSize[1]         = {0};
3995     PetscInt    cones[1]            = {0};
3996     PetscInt    coneOrientations[1] = {0};
3997     PetscScalar vertexCoords[1]     = {0.0};
3998 
3999     PetscCall(DMSetDimension(rdm, 0));
4000     PetscCall(DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4001   } break;
4002   case DM_POLYTOPE_SEGMENT: {
4003     PetscInt    numPoints[2]        = {2, 1};
4004     PetscInt    coneSize[3]         = {2, 0, 0};
4005     PetscInt    cones[2]            = {1, 2};
4006     PetscInt    coneOrientations[2] = {0, 0};
4007     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
4008 
4009     PetscCall(DMSetDimension(rdm, 1));
4010     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4011   } break;
4012   case DM_POLYTOPE_POINT_PRISM_TENSOR: {
4013     PetscInt    numPoints[2]        = {2, 1};
4014     PetscInt    coneSize[3]         = {2, 0, 0};
4015     PetscInt    cones[2]            = {1, 2};
4016     PetscInt    coneOrientations[2] = {0, 0};
4017     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
4018 
4019     PetscCall(DMSetDimension(rdm, 1));
4020     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4021   } break;
4022   case DM_POLYTOPE_TRIANGLE: {
4023     PetscInt    numPoints[2]        = {3, 1};
4024     PetscInt    coneSize[4]         = {3, 0, 0, 0};
4025     PetscInt    cones[3]            = {1, 2, 3};
4026     PetscInt    coneOrientations[3] = {0, 0, 0};
4027     PetscScalar vertexCoords[6]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0};
4028 
4029     PetscCall(DMSetDimension(rdm, 2));
4030     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4031   } break;
4032   case DM_POLYTOPE_QUADRILATERAL: {
4033     PetscInt    numPoints[2]        = {4, 1};
4034     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
4035     PetscInt    cones[4]            = {1, 2, 3, 4};
4036     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
4037     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0};
4038 
4039     PetscCall(DMSetDimension(rdm, 2));
4040     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4041   } break;
4042   case DM_POLYTOPE_SEG_PRISM_TENSOR: {
4043     PetscInt    numPoints[2]        = {4, 1};
4044     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
4045     PetscInt    cones[4]            = {1, 2, 3, 4};
4046     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
4047     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0};
4048 
4049     PetscCall(DMSetDimension(rdm, 2));
4050     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4051   } break;
4052   case DM_POLYTOPE_TETRAHEDRON: {
4053     PetscInt    numPoints[2]        = {4, 1};
4054     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
4055     PetscInt    cones[4]            = {1, 2, 3, 4};
4056     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
4057     PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0};
4058 
4059     PetscCall(DMSetDimension(rdm, 3));
4060     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4061   } break;
4062   case DM_POLYTOPE_HEXAHEDRON: {
4063     PetscInt    numPoints[2]        = {8, 1};
4064     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
4065     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
4066     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
4067     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
4068 
4069     PetscCall(DMSetDimension(rdm, 3));
4070     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4071   } break;
4072   case DM_POLYTOPE_TRI_PRISM: {
4073     PetscInt    numPoints[2]        = {6, 1};
4074     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
4075     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
4076     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
4077     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
4078 
4079     PetscCall(DMSetDimension(rdm, 3));
4080     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4081   } break;
4082   case DM_POLYTOPE_TRI_PRISM_TENSOR: {
4083     PetscInt    numPoints[2]        = {6, 1};
4084     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
4085     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
4086     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
4087     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
4088 
4089     PetscCall(DMSetDimension(rdm, 3));
4090     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4091   } break;
4092   case DM_POLYTOPE_QUAD_PRISM_TENSOR: {
4093     PetscInt    numPoints[2]        = {8, 1};
4094     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
4095     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
4096     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
4097     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
4098 
4099     PetscCall(DMSetDimension(rdm, 3));
4100     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4101   } break;
4102   case DM_POLYTOPE_PYRAMID: {
4103     PetscInt    numPoints[2]        = {5, 1};
4104     PetscInt    coneSize[6]         = {5, 0, 0, 0, 0, 0};
4105     PetscInt    cones[5]            = {1, 2, 3, 4, 5};
4106     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
4107     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, 0.0, 0.0, 1.0};
4108 
4109     PetscCall(DMSetDimension(rdm, 3));
4110     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4111   } break;
4112   default:
4113     SETERRQ(PetscObjectComm((PetscObject)rdm), PETSC_ERR_ARG_WRONG, "Cannot create reference cell for cell type %s", DMPolytopeTypes[ct]);
4114   }
4115   {
4116     PetscInt Nv, v;
4117 
4118     /* Must create the celltype label here so that we do not automatically try to compute the types */
4119     PetscCall(DMCreateLabel(rdm, "celltype"));
4120     PetscCall(DMPlexSetCellType(rdm, 0, ct));
4121     PetscCall(DMPlexGetChart(rdm, NULL, &Nv));
4122     for (v = 1; v < Nv; ++v) PetscCall(DMPlexSetCellType(rdm, v, DM_POLYTOPE_POINT));
4123   }
4124   PetscCall(DMPlexInterpolateInPlace_Internal(rdm));
4125   PetscCall(PetscObjectSetName((PetscObject)rdm, DMPolytopeTypes[ct]));
4126   PetscFunctionReturn(PETSC_SUCCESS);
4127 }
4128 
4129 /*@
4130   DMPlexCreateReferenceCell - Create a `DMPLEX` with the appropriate FEM reference cell
4131 
4132   Collective
4133 
4134   Input Parameters:
4135 + comm - The communicator
4136 - ct   - The cell type of the reference cell
4137 
4138   Output Parameter:
4139 . refdm - The reference cell
4140 
4141   Level: intermediate
4142 
4143 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBoxMesh()`
4144 @*/
4145 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, DMPolytopeType ct, DM *refdm)
4146 {
4147   PetscFunctionBegin;
4148   PetscCall(DMCreate(comm, refdm));
4149   PetscCall(DMSetType(*refdm, DMPLEX));
4150   PetscCall(DMPlexCreateReferenceCell_Internal(*refdm, ct));
4151   PetscFunctionReturn(PETSC_SUCCESS);
4152 }
4153 
4154 static PetscErrorCode DMPlexCreateBoundaryLabel_Private(DM dm, const char name[])
4155 {
4156   DM        plex;
4157   DMLabel   label;
4158   PetscBool hasLabel;
4159 
4160   PetscFunctionBegin;
4161   PetscCall(DMHasLabel(dm, name, &hasLabel));
4162   if (hasLabel) PetscFunctionReturn(PETSC_SUCCESS);
4163   PetscCall(DMCreateLabel(dm, name));
4164   PetscCall(DMGetLabel(dm, name, &label));
4165   PetscCall(DMConvert(dm, DMPLEX, &plex));
4166   PetscCall(DMPlexMarkBoundaryFaces(plex, 1, label));
4167   PetscCall(DMPlexLabelComplete(plex, label));
4168   PetscCall(DMDestroy(&plex));
4169   PetscFunctionReturn(PETSC_SUCCESS);
4170 }
4171 
4172 /*
4173   We use the last coordinate as the radius, the inner radius is lower[dim-1] and the outer radius is upper[dim-1]. Then we map the first coordinate around the circle.
4174 
4175     (x, y) -> (r, theta) = (x[1], (x[0] - lower[0]) * 2\pi/(upper[0] - lower[0]))
4176 */
4177 static void boxToAnnulus(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
4178 {
4179   const PetscReal low = PetscRealPart(constants[0]);
4180   const PetscReal upp = PetscRealPart(constants[1]);
4181   const PetscReal r   = PetscRealPart(u[1]);
4182   const PetscReal th  = 2. * PETSC_PI * (PetscRealPart(u[0]) - low) / (upp - low);
4183 
4184   f0[0] = r * PetscCosReal(th);
4185   f0[1] = r * PetscSinReal(th);
4186 }
4187 
4188 // Insert vertices and their joins, marked by depth
4189 static PetscErrorCode ProcessCohesiveLabel_Vertices(DM dm, DMLabel label, DMLabel vlabel, PetscInt val, PetscInt n, const PetscInt vertices[])
4190 {
4191   PetscFunctionBegin;
4192   PetscCall(DMPlexMarkSubmesh_Interpolated(dm, vlabel, val, PETSC_FALSE, PETSC_FALSE, label, NULL));
4193   PetscFunctionReturn(PETSC_SUCCESS);
4194 }
4195 
4196 // Insert faces and their closures, marked by depth
4197 static PetscErrorCode ProcessCohesiveLabel_Faces(DM dm, DMLabel label, PetscInt n, const PetscInt faces[])
4198 {
4199   PetscFunctionBegin;
4200   for (PetscInt p = 0; p < n; ++p) {
4201     const PetscInt point   = faces[p];
4202     PetscInt      *closure = NULL;
4203     PetscInt       clSize, pdepth;
4204 
4205     PetscCall(DMPlexGetPointDepth(dm, point, &pdepth));
4206     PetscCall(DMLabelSetValue(label, point, pdepth));
4207     PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &clSize, &closure));
4208     for (PetscInt cl = 0; cl < clSize * 2; cl += 2) {
4209       PetscCall(DMPlexGetPointDepth(dm, closure[cl], &pdepth));
4210       PetscCall(DMLabelSetValue(label, closure[cl], pdepth));
4211     }
4212     PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &clSize, &closure));
4213   }
4214   PetscFunctionReturn(PETSC_SUCCESS);
4215 }
4216 
4217 PETSC_EXTERN PetscErrorCode PetscOptionsFindPairPrefix_Private(PetscOptions, const char pre[], const char name[], const char *option[], const char *value[], PetscBool *flg);
4218 
4219 const char *const DMPlexShapes[] = {"box", "box_surface", "ball", "sphere", "cylinder", "schwarz_p", "gyroid", "doublet", "annulus", "hypercubic", "zbox", "unknown", "DMPlexShape", "DM_SHAPE_", NULL};
4220 
4221 static PetscErrorCode DMPlexCreateFromOptions_Internal(PetscOptionItems *PetscOptionsObject, PetscBool *useCoordSpace, DM dm)
4222 {
4223   DMPlexShape    shape   = DM_SHAPE_BOX;
4224   DMPolytopeType cell    = DM_POLYTOPE_TRIANGLE;
4225   PetscInt       dim     = 2;
4226   PetscBool      simplex = PETSC_TRUE, interpolate = PETSC_TRUE, orient = PETSC_FALSE, adjCone = PETSC_FALSE, adjClosure = PETSC_TRUE, refDomain = PETSC_FALSE;
4227   PetscBool      flg, flg2, fflg, strflg, bdfflg, nameflg;
4228   MPI_Comm       comm;
4229   char           filename[PETSC_MAX_PATH_LEN]   = "<unspecified>";
4230   char           bdFilename[PETSC_MAX_PATH_LEN] = "<unspecified>";
4231   char           plexname[PETSC_MAX_PATH_LEN]   = "";
4232   const char    *option;
4233 
4234   PetscFunctionBegin;
4235   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromOptions, dm, 0, 0, 0));
4236   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
4237   /* TODO Turn this into a registration interface */
4238   PetscCall(PetscOptionsString("-dm_plex_filename", "File containing a mesh", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &fflg));
4239   PetscCall(PetscOptionsString("-dm_plex_file_contents", "Contents of a file format in a string", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &strflg));
4240   PetscCall(PetscOptionsString("-dm_plex_boundary_filename", "File containing a mesh boundary", "DMPlexCreateFromFile", bdFilename, bdFilename, sizeof(bdFilename), &bdfflg));
4241   PetscCall(PetscOptionsString("-dm_plex_name", "Name of the mesh in the file", "DMPlexCreateFromFile", plexname, plexname, sizeof(plexname), &nameflg));
4242   PetscCall(PetscOptionsEnum("-dm_plex_cell", "Cell shape", "", DMPolytopeTypes, (PetscEnum)cell, (PetscEnum *)&cell, NULL));
4243   PetscCall(PetscOptionsBool("-dm_plex_reference_cell_domain", "Use a reference cell domain", "", refDomain, &refDomain, NULL));
4244   PetscCall(PetscOptionsEnum("-dm_plex_shape", "Shape for built-in mesh", "", DMPlexShapes, (PetscEnum)shape, (PetscEnum *)&shape, &flg));
4245   PetscCall(PetscOptionsBoundedInt("-dm_plex_dim", "Topological dimension of the mesh", "DMGetDimension", dim, &dim, &flg, 0));
4246   PetscCall(PetscOptionsBool("-dm_plex_simplex", "Mesh cell shape", "", simplex, &simplex, &flg));
4247   PetscCall(PetscOptionsBool("-dm_plex_interpolate", "Flag to create edges and faces automatically", "", interpolate, &interpolate, &flg));
4248   PetscCall(PetscOptionsBool("-dm_plex_orient", "Orient the constructed mesh", "DMPlexOrient", orient, &orient, &flg));
4249   PetscCall(PetscOptionsBool("-dm_plex_adj_cone", "Set adjacency direction", "DMSetBasicAdjacency", adjCone, &adjCone, &flg));
4250   PetscCall(PetscOptionsBool("-dm_plex_adj_closure", "Set adjacency size", "DMSetBasicAdjacency", adjClosure, &adjClosure, &flg2));
4251   if (flg || flg2) PetscCall(DMSetBasicAdjacency(dm, adjCone, adjClosure));
4252 
4253   switch (cell) {
4254   case DM_POLYTOPE_POINT:
4255   case DM_POLYTOPE_SEGMENT:
4256   case DM_POLYTOPE_POINT_PRISM_TENSOR:
4257   case DM_POLYTOPE_TRIANGLE:
4258   case DM_POLYTOPE_QUADRILATERAL:
4259   case DM_POLYTOPE_TETRAHEDRON:
4260   case DM_POLYTOPE_HEXAHEDRON:
4261     *useCoordSpace = PETSC_TRUE;
4262     break;
4263   default:
4264     *useCoordSpace = PETSC_FALSE;
4265     break;
4266   }
4267 
4268   if (fflg) {
4269     DM          dmnew;
4270     const char *name;
4271 
4272     PetscCall(PetscObjectGetName((PetscObject)dm, &name));
4273     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), filename, nameflg ? plexname : name, interpolate, &dmnew));
4274     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4275     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4276   } else if (refDomain) {
4277     PetscCall(DMPlexCreateReferenceCell_Internal(dm, cell));
4278   } else if (bdfflg) {
4279     DM          bdm, dmnew;
4280     const char *name;
4281 
4282     PetscCall(PetscObjectGetName((PetscObject)dm, &name));
4283     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), bdFilename, nameflg ? plexname : name, interpolate, &bdm));
4284     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)bdm, "bd_"));
4285     PetscCall(DMSetFromOptions(bdm));
4286     PetscCall(DMPlexGenerate(bdm, NULL, interpolate, &dmnew));
4287     PetscCall(DMDestroy(&bdm));
4288     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4289     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4290   } else if (strflg) {
4291     DM          dmnew;
4292     PetscViewer viewer;
4293     const char *contents;
4294     char       *strname;
4295     char        tmpdir[PETSC_MAX_PATH_LEN];
4296     char        tmpfilename[PETSC_MAX_PATH_LEN];
4297     char        name[PETSC_MAX_PATH_LEN];
4298     MPI_Comm    comm;
4299     PetscMPIInt rank;
4300 
4301     PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
4302     PetscCallMPI(MPI_Comm_rank(comm, &rank));
4303     PetscCall(PetscStrchr(filename, ':', &strname));
4304     PetscCheck(strname, comm, PETSC_ERR_ARG_WRONG, "File contents must have the form \"ext:string_name\", not %s", filename);
4305     strname[0] = '\0';
4306     ++strname;
4307     PetscCall(PetscDLSym(NULL, strname, (void **)&contents));
4308     PetscCheck(contents, comm, PETSC_ERR_ARG_WRONG, "Could not locate mesh string %s", strname);
4309     PetscCall(PetscGetTmp(comm, tmpdir, PETSC_MAX_PATH_LEN));
4310     PetscCall(PetscStrlcat(tmpdir, "/meshXXXXXX", PETSC_MAX_PATH_LEN));
4311     PetscCall(PetscMkdtemp(tmpdir));
4312     PetscCall(PetscSNPrintf(tmpfilename, PETSC_MAX_PATH_LEN, "%s/mesh.%s", tmpdir, filename));
4313     PetscCall(PetscViewerASCIIOpen(comm, tmpfilename, &viewer));
4314     PetscCall(PetscViewerASCIIPrintf(viewer, "%s\n", contents));
4315     PetscCall(PetscViewerDestroy(&viewer));
4316     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), tmpfilename, plexname, interpolate, &dmnew));
4317     PetscCall(PetscRMTree(tmpdir));
4318     PetscCall(PetscSNPrintf(name, PETSC_MAX_PATH_LEN, "%s Mesh", strname));
4319     PetscCall(PetscObjectSetName((PetscObject)dm, name));
4320     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4321     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4322   } else {
4323     PetscCall(PetscObjectSetName((PetscObject)dm, DMPlexShapes[shape]));
4324     switch (shape) {
4325     case DM_SHAPE_BOX:
4326     case DM_SHAPE_ZBOX:
4327     case DM_SHAPE_ANNULUS: {
4328       PetscInt       faces[3]  = {0, 0, 0};
4329       PetscReal      lower[3]  = {0, 0, 0};
4330       PetscReal      upper[3]  = {1, 1, 1};
4331       DMBoundaryType bdt[3]    = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
4332       PetscBool      isAnnular = shape == DM_SHAPE_ANNULUS ? PETSC_TRUE : PETSC_FALSE;
4333       PetscInt       i, n;
4334 
4335       n = dim;
4336       for (i = 0; i < dim; ++i) faces[i] = (dim == 1 ? 1 : 4 - dim);
4337       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
4338       n = 3;
4339       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
4340       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4341       n = 3;
4342       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
4343       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4344       n = 3;
4345       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
4346       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4347 
4348       PetscCheck(!isAnnular || dim == 2, comm, PETSC_ERR_ARG_OUTOFRANGE, "Only two dimensional annuli have been implemented");
4349       if (isAnnular)
4350         for (i = 0; i < dim - 1; ++i) bdt[i] = DM_BOUNDARY_PERIODIC;
4351 
4352       switch (cell) {
4353       case DM_POLYTOPE_TRI_PRISM_TENSOR:
4354         PetscCall(DMPlexCreateWedgeBoxMesh_Internal(dm, faces, lower, upper, bdt));
4355         if (!interpolate) {
4356           DM udm;
4357 
4358           PetscCall(DMPlexUninterpolate(dm, &udm));
4359           PetscCall(DMPlexReplace_Internal(dm, &udm));
4360         }
4361         break;
4362       default:
4363         PetscCall(DMPlexCreateBoxMesh_Internal(dm, shape, dim, simplex, faces, lower, upper, bdt, interpolate));
4364         break;
4365       }
4366       if (isAnnular) {
4367         DM          cdm;
4368         PetscDS     cds;
4369         PetscScalar bounds[2] = {lower[0], upper[0]};
4370 
4371         // Fix coordinates for annular region
4372         PetscCall(DMSetPeriodicity(dm, NULL, NULL, NULL));
4373         PetscCall(DMSetCellCoordinatesLocal(dm, NULL));
4374         PetscCall(DMSetCellCoordinates(dm, NULL));
4375         PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_TRUE, NULL));
4376         PetscCall(DMGetCoordinateDM(dm, &cdm));
4377         PetscCall(DMGetDS(cdm, &cds));
4378         PetscCall(PetscDSSetConstants(cds, 2, bounds));
4379         PetscCall(DMPlexRemapGeometry(dm, 0.0, boxToAnnulus));
4380       }
4381     } break;
4382     case DM_SHAPE_BOX_SURFACE: {
4383       PetscInt  faces[3] = {0, 0, 0};
4384       PetscReal lower[3] = {0, 0, 0};
4385       PetscReal upper[3] = {1, 1, 1};
4386       PetscInt  i, n;
4387 
4388       n = dim + 1;
4389       for (i = 0; i < dim + 1; ++i) faces[i] = (dim + 1 == 1 ? 1 : 4 - (dim + 1));
4390       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
4391       n = 3;
4392       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
4393       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
4394       n = 3;
4395       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
4396       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
4397       PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(dm, dim + 1, faces, lower, upper, interpolate));
4398     } break;
4399     case DM_SHAPE_SPHERE: {
4400       PetscReal R = 1.0;
4401 
4402       PetscCall(PetscOptionsReal("-dm_plex_sphere_radius", "Radius of the sphere", "", R, &R, &flg));
4403       PetscCall(DMPlexCreateSphereMesh_Internal(dm, dim, simplex, R));
4404     } break;
4405     case DM_SHAPE_BALL: {
4406       PetscReal R = 1.0;
4407 
4408       PetscCall(PetscOptionsReal("-dm_plex_ball_radius", "Radius of the ball", "", R, &R, &flg));
4409       PetscCall(DMPlexCreateBallMesh_Internal(dm, dim, R));
4410     } break;
4411     case DM_SHAPE_CYLINDER: {
4412       DMBoundaryType bdt = DM_BOUNDARY_NONE;
4413       PetscInt       Nw  = 6;
4414       PetscInt       Nr  = 0;
4415 
4416       PetscCall(PetscOptionsEnum("-dm_plex_cylinder_bd", "Boundary type in the z direction", "", DMBoundaryTypes, (PetscEnum)bdt, (PetscEnum *)&bdt, NULL));
4417       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_wedges", "Number of wedges around the cylinder", "", Nw, &Nw, NULL));
4418       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_refine", "Number of refinements before projection", "", Nr, &Nr, NULL));
4419       switch (cell) {
4420       case DM_POLYTOPE_TRI_PRISM_TENSOR:
4421         PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(dm, Nw, interpolate));
4422         break;
4423       default:
4424         PetscCall(DMPlexCreateHexCylinderMesh_Internal(dm, bdt, Nr));
4425         break;
4426       }
4427     } break;
4428     case DM_SHAPE_SCHWARZ_P: // fallthrough
4429     case DM_SHAPE_GYROID: {
4430       PetscInt       extent[3] = {1, 1, 1}, refine = 0, layers = 0, three;
4431       PetscReal      thickness   = 0.;
4432       DMBoundaryType periodic[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
4433       DMPlexTPSType  tps_type    = shape == DM_SHAPE_SCHWARZ_P ? DMPLEX_TPS_SCHWARZ_P : DMPLEX_TPS_GYROID;
4434       PetscBool      tps_distribute;
4435       PetscCall(PetscOptionsIntArray("-dm_plex_tps_extent", "Number of replicas for each of three dimensions", NULL, extent, (three = 3, &three), NULL));
4436       PetscCall(PetscOptionsInt("-dm_plex_tps_refine", "Number of refinements", NULL, refine, &refine, NULL));
4437       PetscCall(PetscOptionsEnumArray("-dm_plex_tps_periodic", "Periodicity in each of three dimensions", NULL, DMBoundaryTypes, (PetscEnum *)periodic, (three = 3, &three), NULL));
4438       PetscCall(PetscOptionsInt("-dm_plex_tps_layers", "Number of layers in volumetric extrusion (or zero to not extrude)", NULL, layers, &layers, NULL));
4439       PetscCall(PetscOptionsReal("-dm_plex_tps_thickness", "Thickness of volumetric extrusion", NULL, thickness, &thickness, NULL));
4440       PetscCall(DMPlexDistributeGetDefault(dm, &tps_distribute));
4441       PetscCall(PetscOptionsBool("-dm_plex_tps_distribute", "Distribute the 2D mesh prior to refinement and extrusion", NULL, tps_distribute, &tps_distribute, NULL));
4442       PetscCall(DMPlexCreateTPSMesh_Internal(dm, tps_type, extent, periodic, tps_distribute, refine, layers, thickness));
4443     } break;
4444     case DM_SHAPE_DOUBLET: {
4445       DM        dmnew;
4446       PetscReal rl = 0.0;
4447 
4448       PetscCall(PetscOptionsReal("-dm_plex_doublet_refinementlimit", "Refinement limit", NULL, rl, &rl, NULL));
4449       PetscCall(DMPlexCreateDoublet(PetscObjectComm((PetscObject)dm), dim, simplex, interpolate, rl, &dmnew));
4450       PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4451       PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4452     } break;
4453     case DM_SHAPE_HYPERCUBIC: {
4454       PetscInt       *edges;
4455       PetscReal      *lower, *upper;
4456       DMBoundaryType *bdt;
4457       PetscInt        n, d;
4458 
4459       *useCoordSpace = PETSC_FALSE;
4460       PetscCall(PetscMalloc4(dim, &edges, dim, &lower, dim, &upper, dim, &bdt));
4461       for (d = 0; d < dim; ++d) {
4462         edges[d] = 1;
4463         lower[d] = 0.;
4464         upper[d] = 1.;
4465         bdt[d]   = DM_BOUNDARY_PERIODIC;
4466       }
4467       n = dim;
4468       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", edges, &n, &flg));
4469       n = dim;
4470       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
4471       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4472       n = dim;
4473       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
4474       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4475       n = dim;
4476       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
4477       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4478       PetscCall(DMPlexCreateHypercubicMesh_Internal(dm, dim, lower, upper, edges, bdt));
4479       PetscCall(PetscFree4(edges, lower, upper, bdt));
4480     } break;
4481     default:
4482       SETERRQ(comm, PETSC_ERR_SUP, "Domain shape %s is unsupported", DMPlexShapes[shape]);
4483     }
4484   }
4485   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
4486   if (!((PetscObject)dm)->name && nameflg) PetscCall(PetscObjectSetName((PetscObject)dm, plexname));
4487   if (orient) PetscCall(DMPlexOrient(dm));
4488   // Allow label creation
4489   PetscCall(PetscOptionsFindPairPrefix_Private(NULL, ((PetscObject)dm)->prefix, "-dm_plex_label_", &option, NULL, &flg));
4490   if (flg) {
4491     DMLabel     label;
4492     PetscInt    points[1024], n = 1024;
4493     char        fulloption[PETSC_MAX_PATH_LEN];
4494     const char *name = &option[14];
4495 
4496     PetscCall(DMCreateLabel(dm, name));
4497     PetscCall(DMGetLabel(dm, name, &label));
4498     fulloption[0] = '-';
4499     fulloption[1] = 0;
4500     PetscCall(PetscStrlcat(fulloption, option, PETSC_MAX_PATH_LEN));
4501     PetscCall(PetscOptionsGetIntArray(NULL, ((PetscObject)dm)->prefix, fulloption, points, &n, NULL));
4502     for (PetscInt p = 0; p < n; ++p) PetscCall(DMLabelSetValue(label, points[p], 1));
4503   }
4504   // Allow cohesive label creation
4505   //   Faces are input, completed, and all points are marked with their depth
4506   PetscCall(PetscOptionsFindPairPrefix_Private(NULL, ((PetscObject)dm)->prefix, "-dm_plex_cohesive_label_", &option, NULL, &flg));
4507   if (flg) {
4508     DMLabel   label;
4509     PetscInt  points[1024], n, pStart, pEnd, Nl = 1;
4510     PetscBool noCreate = PETSC_FALSE;
4511     char      fulloption[PETSC_MAX_PATH_LEN];
4512     char      name[PETSC_MAX_PATH_LEN];
4513     size_t    len;
4514 
4515     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
4516     PetscCall(PetscStrncpy(name, &option[23], PETSC_MAX_PATH_LEN));
4517     PetscCall(PetscStrlen(name, &len));
4518     if (name[len - 1] == '0') Nl = 10;
4519     for (PetscInt l = 0; l < Nl; ++l) {
4520       if (l > 0) name[len - 1] = (char)('0' + l);
4521       fulloption[0] = 0;
4522       PetscCall(PetscStrlcat(fulloption, "-dm_plex_cohesive_label_", 32));
4523       PetscCall(PetscStrlcat(fulloption, name, PETSC_MAX_PATH_LEN - 32));
4524       n = 1024;
4525       PetscCall(PetscOptionsGetIntArray(NULL, ((PetscObject)dm)->prefix, fulloption, points, &n, &flg));
4526       if (!flg) break;
4527       PetscCall(DMHasLabel(dm, name, &noCreate));
4528       if (noCreate) {
4529         DMLabel         inlabel;
4530         IS              pointIS;
4531         const PetscInt *lpoints;
4532         PetscInt        pdep, ln, inval = points[0];
4533         char            newname[PETSC_MAX_PATH_LEN];
4534 
4535         PetscCheck(n == 1, comm, PETSC_ERR_ARG_WRONG, "Must specify a label value with this option");
4536         PetscCall(DMGetLabel(dm, name, &inlabel));
4537         PetscCall(DMLabelGetStratumIS(inlabel, inval, &pointIS));
4538         PetscCall(ISGetLocalSize(pointIS, &ln));
4539         PetscCall(ISGetIndices(pointIS, &lpoints));
4540         PetscCall(DMPlexGetPointDepth(dm, lpoints[0], &pdep));
4541         PetscCall(PetscSNPrintf(newname, PETSC_MAX_PATH_LEN, "%s%" PetscInt_FMT, name, points[0]));
4542         PetscCall(DMCreateLabel(dm, newname));
4543         PetscCall(DMGetLabel(dm, newname, &label));
4544         if (!pdep) PetscCall(ProcessCohesiveLabel_Vertices(dm, label, inlabel, inval, ln, lpoints));
4545         else PetscCall(ProcessCohesiveLabel_Faces(dm, label, ln, lpoints));
4546         PetscCall(ISRestoreIndices(pointIS, &lpoints));
4547         PetscCall(ISDestroy(&pointIS));
4548       } else {
4549         PetscCall(DMCreateLabel(dm, name));
4550         PetscCall(DMGetLabel(dm, name, &label));
4551         if (pStart >= pEnd) n = 0;
4552         PetscCall(ProcessCohesiveLabel_Faces(dm, label, n, points));
4553       }
4554       PetscCall(DMPlexOrientLabel(dm, label));
4555       PetscCall(DMPlexLabelCohesiveComplete(dm, label, NULL, 1, PETSC_FALSE, PETSC_FALSE, NULL));
4556     }
4557   }
4558   PetscCall(DMViewFromOptions(dm, NULL, "-created_dm_view"));
4559   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromOptions, dm, 0, 0, 0));
4560   PetscFunctionReturn(PETSC_SUCCESS);
4561 }
4562 
4563 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
4564 {
4565   DM_Plex  *mesh = (DM_Plex *)dm->data;
4566   PetscBool flg, flg2;
4567   char      bdLabel[PETSC_MAX_PATH_LEN];
4568   char      method[PETSC_MAX_PATH_LEN];
4569 
4570   PetscFunctionBegin;
4571   /* Handle viewing */
4572   PetscCall(PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL));
4573   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fem", "Debug output level for all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL, 0));
4574   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fvm", "Debug output level for all fvm computations", "DMPlexSNESComputeResidualFVM", 0, &mesh->printFVM, NULL, 0));
4575   PetscCall(PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL));
4576   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL, 0));
4577   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_locate", "Debug output level all point location computations", "DMLocatePoints", 0, &mesh->printLocate, NULL, 0));
4578   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_project", "Debug output level all projection computations", "DMPlexProject", 0, &mesh->printProject, NULL, 0));
4579   PetscCall(DMMonitorSetFromOptions(dm, "-dm_plex_monitor_throughput", "Monitor the simulation throughput", "DMPlexMonitorThroughput", DMPlexMonitorThroughput, NULL, &flg));
4580   if (flg) PetscCall(PetscLogDefaultBegin());
4581   /* Labeling */
4582   PetscCall(PetscOptionsString("-dm_plex_boundary_label", "Label to mark the mesh boundary", "", bdLabel, bdLabel, sizeof(bdLabel), &flg));
4583   if (flg) PetscCall(DMPlexCreateBoundaryLabel_Private(dm, bdLabel));
4584   /* Point Location */
4585   PetscCall(PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL));
4586   /* Partitioning and distribution */
4587   PetscCall(PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL));
4588   /* Reordering */
4589   PetscCall(PetscOptionsBool("-dm_reorder_section", "Compute point permutation for local section", "DMReorderSectionSetDefault", PETSC_FALSE, &flg2, &flg));
4590   if (flg) PetscCall(DMReorderSectionSetDefault(dm, flg2 ? DM_REORDER_DEFAULT_TRUE : DM_REORDER_DEFAULT_FALSE));
4591   PetscCall(PetscOptionsString("-dm_reorder_section_type", "Reordering method for local section", "DMReorderSectionSetType", method, method, PETSC_MAX_PATH_LEN, &flg));
4592   if (flg) PetscCall(DMReorderSectionSetType(dm, method));
4593   /* Generation and remeshing */
4594   PetscCall(PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL));
4595   /* Projection behavior */
4596   PetscCall(PetscOptionsBoundedInt("-dm_plex_max_projection_height", "Maximum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL, 0));
4597   PetscCall(PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL));
4598   /* Checking structure */
4599   {
4600     PetscBool all = PETSC_FALSE;
4601 
4602     PetscCall(PetscOptionsBool("-dm_plex_check_all", "Perform all basic checks", "DMPlexCheck", PETSC_FALSE, &all, NULL));
4603     if (all) {
4604       PetscCall(DMPlexCheck(dm));
4605     } else {
4606       PetscCall(PetscOptionsBool("-dm_plex_check_symmetry", "Check that the adjacency information in the mesh is symmetric", "DMPlexCheckSymmetry", PETSC_FALSE, &flg, &flg2));
4607       if (flg && flg2) PetscCall(DMPlexCheckSymmetry(dm));
4608       PetscCall(PetscOptionsBool("-dm_plex_check_skeleton", "Check that each cell has the correct number of vertices (only for homogeneous simplex or tensor meshes)", "DMPlexCheckSkeleton", PETSC_FALSE, &flg, &flg2));
4609       if (flg && flg2) PetscCall(DMPlexCheckSkeleton(dm, 0));
4610       PetscCall(PetscOptionsBool("-dm_plex_check_faces", "Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type", "DMPlexCheckFaces", PETSC_FALSE, &flg, &flg2));
4611       if (flg && flg2) PetscCall(DMPlexCheckFaces(dm, 0));
4612       PetscCall(PetscOptionsBool("-dm_plex_check_geometry", "Check that cells have positive volume", "DMPlexCheckGeometry", PETSC_FALSE, &flg, &flg2));
4613       if (flg && flg2) PetscCall(DMPlexCheckGeometry(dm));
4614       PetscCall(PetscOptionsBool("-dm_plex_check_pointsf", "Check some necessary conditions for PointSF", "DMPlexCheckPointSF", PETSC_FALSE, &flg, &flg2));
4615       if (flg && flg2) PetscCall(DMPlexCheckPointSF(dm, NULL, PETSC_FALSE));
4616       PetscCall(PetscOptionsBool("-dm_plex_check_interface_cones", "Check points on inter-partition interfaces have conforming order of cone points", "DMPlexCheckInterfaceCones", PETSC_FALSE, &flg, &flg2));
4617       if (flg && flg2) PetscCall(DMPlexCheckInterfaceCones(dm));
4618     }
4619     PetscCall(PetscOptionsBool("-dm_plex_check_cell_shape", "Check cell shape", "DMPlexCheckCellShape", PETSC_FALSE, &flg, &flg2));
4620     if (flg && flg2) PetscCall(DMPlexCheckCellShape(dm, PETSC_TRUE, PETSC_DETERMINE));
4621   }
4622   {
4623     PetscReal scale = 1.0;
4624 
4625     PetscCall(PetscOptionsReal("-dm_plex_scale", "Scale factor for mesh coordinates", "DMPlexScale", scale, &scale, &flg));
4626     if (flg) {
4627       Vec coordinates, coordinatesLocal;
4628 
4629       PetscCall(DMGetCoordinates(dm, &coordinates));
4630       PetscCall(DMGetCoordinatesLocal(dm, &coordinatesLocal));
4631       PetscCall(VecScale(coordinates, scale));
4632       PetscCall(VecScale(coordinatesLocal, scale));
4633     }
4634   }
4635   PetscCall(PetscPartitionerSetFromOptions(mesh->partitioner));
4636   PetscFunctionReturn(PETSC_SUCCESS);
4637 }
4638 
4639 PetscErrorCode DMSetFromOptions_Overlap_Plex(DM dm, PetscOptionItems *PetscOptionsObject, PetscInt *overlap)
4640 {
4641   PetscInt  numOvLabels = 16, numOvExLabels = 16;
4642   char     *ovLabelNames[16], *ovExLabelNames[16];
4643   PetscInt  numOvValues = 16, numOvExValues = 16, l;
4644   PetscBool flg;
4645 
4646   PetscFunctionBegin;
4647   PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", *overlap, overlap, NULL, 0));
4648   PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_labels", "List of overlap label names", "DMPlexDistribute", ovLabelNames, &numOvLabels, &flg));
4649   if (!flg) numOvLabels = 0;
4650   if (numOvLabels) {
4651     ((DM_Plex *)dm->data)->numOvLabels = numOvLabels;
4652     for (l = 0; l < numOvLabels; ++l) {
4653       PetscCall(DMGetLabel(dm, ovLabelNames[l], &((DM_Plex *)dm->data)->ovLabels[l]));
4654       PetscCheck(((DM_Plex *)dm->data)->ovLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovLabelNames[l]);
4655       PetscCall(PetscFree(ovLabelNames[l]));
4656     }
4657     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_values", "List of overlap label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovValues, &numOvValues, &flg));
4658     if (!flg) numOvValues = 0;
4659     PetscCheck(numOvLabels == numOvValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvLabels, numOvValues);
4660 
4661     PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_exclude_labels", "List of overlap exclude label names", "DMPlexDistribute", ovExLabelNames, &numOvExLabels, &flg));
4662     if (!flg) numOvExLabels = 0;
4663     ((DM_Plex *)dm->data)->numOvExLabels = numOvExLabels;
4664     for (l = 0; l < numOvExLabels; ++l) {
4665       PetscCall(DMGetLabel(dm, ovExLabelNames[l], &((DM_Plex *)dm->data)->ovExLabels[l]));
4666       PetscCheck(((DM_Plex *)dm->data)->ovExLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovExLabelNames[l]);
4667       PetscCall(PetscFree(ovExLabelNames[l]));
4668     }
4669     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_exclude_values", "List of overlap exclude label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovExValues, &numOvExValues, &flg));
4670     if (!flg) numOvExValues = 0;
4671     PetscCheck(numOvExLabels == numOvExValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of exclude labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvExLabels, numOvExValues);
4672   }
4673   PetscFunctionReturn(PETSC_SUCCESS);
4674 }
4675 
4676 static PetscErrorCode DMSetFromOptions_Plex(DM dm, PetscOptionItems *PetscOptionsObject)
4677 {
4678   PetscFunctionList    ordlist;
4679   char                 oname[256];
4680   char                 sublabelname[PETSC_MAX_PATH_LEN] = "";
4681   DMReorderDefaultFlag reorder;
4682   PetscReal            volume    = -1.0;
4683   PetscInt             prerefine = 0, refine = 0, r, coarsen = 0, overlap = 0, extLayers = 0, dim;
4684   PetscBool            uniformOrig = PETSC_FALSE, created = PETSC_FALSE, uniform = PETSC_TRUE, distribute, saveSF = PETSC_FALSE, interpolate = PETSC_TRUE, coordSpace = PETSC_TRUE, remap = PETSC_TRUE, ghostCells = PETSC_FALSE, isHierarchy, flg;
4685 
4686   PetscFunctionBegin;
4687   PetscOptionsHeadBegin(PetscOptionsObject, "DMPlex Options");
4688   if (dm->cloneOpts) goto non_refine;
4689   /* Handle automatic creation */
4690   PetscCall(DMGetDimension(dm, &dim));
4691   if (dim < 0) {
4692     PetscCall(DMPlexCreateFromOptions_Internal(PetscOptionsObject, &coordSpace, dm));
4693     created = PETSC_TRUE;
4694   }
4695   PetscCall(DMGetDimension(dm, &dim));
4696   /* Handle interpolation before distribution */
4697   PetscCall(PetscOptionsBool("-dm_plex_interpolate_pre", "Flag to interpolate mesh before distribution", "", interpolate, &interpolate, &flg));
4698   if (flg) {
4699     DMPlexInterpolatedFlag interpolated;
4700 
4701     PetscCall(DMPlexIsInterpolated(dm, &interpolated));
4702     if (interpolated == DMPLEX_INTERPOLATED_FULL && !interpolate) {
4703       DM udm;
4704 
4705       PetscCall(DMPlexUninterpolate(dm, &udm));
4706       PetscCall(DMPlexReplace_Internal(dm, &udm));
4707     } else if (interpolated != DMPLEX_INTERPOLATED_FULL && interpolate) {
4708       DM idm;
4709 
4710       PetscCall(DMPlexInterpolate(dm, &idm));
4711       PetscCall(DMPlexReplace_Internal(dm, &idm));
4712     }
4713   }
4714   // Handle submesh selection before distribution
4715   PetscCall(PetscOptionsString("-dm_plex_submesh", "Label to use for submesh selection", "", sublabelname, sublabelname, PETSC_MAX_PATH_LEN, &flg));
4716   if (flg) {
4717     DM              subdm;
4718     DMLabel         label;
4719     IS              valueIS, pointIS;
4720     const PetscInt *values, *points;
4721     PetscBool       markedFaces = PETSC_FALSE;
4722     PetscInt        Nv, value, Np;
4723 
4724     PetscCall(DMGetLabel(dm, sublabelname, &label));
4725     PetscCall(DMLabelGetNumValues(label, &Nv));
4726     PetscCheck(Nv == 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Only a single label value is currently supported for submesh selection, not %" PetscInt_FMT, Nv);
4727     PetscCall(DMLabelGetValueIS(label, &valueIS));
4728     PetscCall(ISGetIndices(valueIS, &values));
4729     value = values[0];
4730     PetscCall(ISRestoreIndices(valueIS, &values));
4731     PetscCall(ISDestroy(&valueIS));
4732     PetscCall(DMLabelGetStratumSize(label, value, &Np));
4733     PetscCall(DMLabelGetStratumIS(label, value, &pointIS));
4734     PetscCall(ISGetIndices(pointIS, &points));
4735     for (PetscInt p = 0; p < Np; ++p) {
4736       PetscInt pdepth;
4737 
4738       PetscCall(DMPlexGetPointDepth(dm, points[p], &pdepth));
4739       if (pdepth) {
4740         markedFaces = PETSC_TRUE;
4741         break;
4742       }
4743     }
4744     PetscCall(ISRestoreIndices(pointIS, &points));
4745     PetscCall(ISDestroy(&pointIS));
4746     PetscCall(DMPlexCreateSubmesh(dm, label, value, markedFaces, &subdm));
4747     PetscCall(DMPlexReplace_Internal(dm, &subdm));
4748     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4749   }
4750   /* Handle DMPlex refinement before distribution */
4751   PetscCall(DMPlexGetRefinementUniform(dm, &uniformOrig));
4752   PetscCall(PetscOptionsBoundedInt("-dm_refine_pre", "The number of refinements before distribution", "DMCreate", prerefine, &prerefine, NULL, 0));
4753   PetscCall(PetscOptionsBool("-dm_refine_remap_pre", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
4754   PetscCall(PetscOptionsBool("-dm_refine_uniform_pre", "Flag for uniform refinement before distribution", "DMCreate", uniform, &uniform, &flg));
4755   if (flg) PetscCall(DMPlexSetRefinementUniform(dm, uniform));
4756   PetscCall(PetscOptionsReal("-dm_refine_volume_limit_pre", "The maximum cell volume after refinement before distribution", "DMCreate", volume, &volume, &flg));
4757   if (flg) {
4758     PetscCall(DMPlexSetRefinementUniform(dm, PETSC_FALSE));
4759     PetscCall(DMPlexSetRefinementLimit(dm, volume));
4760     prerefine = PetscMax(prerefine, 1);
4761   }
4762   if (prerefine) PetscCall(DMLocalizeCoordinates(dm));
4763   for (r = 0; r < prerefine; ++r) {
4764     DM             rdm;
4765     PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4766 
4767     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4768     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
4769     PetscCall(DMPlexReplace_Internal(dm, &rdm));
4770     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4771     if (coordFunc && remap) {
4772       PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4773       ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4774     }
4775   }
4776   PetscCall(DMPlexSetRefinementUniform(dm, uniformOrig));
4777   /* Handle DMPlex extrusion before distribution */
4778   PetscCall(PetscOptionsBoundedInt("-dm_extrude", "The number of layers to extrude", "", extLayers, &extLayers, NULL, 0));
4779   if (extLayers) {
4780     DM edm;
4781 
4782     PetscCall(DMExtrude(dm, extLayers, &edm));
4783     PetscCall(DMPlexReplace_Internal(dm, &edm));
4784     ((DM_Plex *)dm->data)->coordFunc = NULL;
4785     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4786     extLayers = 0;
4787     PetscCall(DMGetDimension(dm, &dim));
4788   }
4789   /* Handle DMPlex reordering before distribution */
4790   PetscCall(DMPlexReorderGetDefault(dm, &reorder));
4791   PetscCall(MatGetOrderingList(&ordlist));
4792   PetscCall(PetscStrncpy(oname, MATORDERINGNATURAL, sizeof(oname)));
4793   PetscCall(PetscOptionsFList("-dm_plex_reorder", "Set mesh reordering type", "DMPlexGetOrdering", ordlist, MATORDERINGNATURAL, oname, sizeof(oname), &flg));
4794   if (reorder == DM_REORDER_DEFAULT_TRUE || flg) {
4795     DM pdm;
4796     IS perm;
4797 
4798     PetscCall(DMPlexGetOrdering(dm, oname, NULL, &perm));
4799     PetscCall(DMPlexPermute(dm, perm, &pdm));
4800     PetscCall(ISDestroy(&perm));
4801     PetscCall(DMPlexReplace_Internal(dm, &pdm));
4802     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4803   }
4804   /* Handle DMPlex distribution */
4805   PetscCall(DMPlexDistributeGetDefault(dm, &distribute));
4806   PetscCall(PetscOptionsBool("-dm_distribute", "Flag to redistribute a mesh among processes", "DMPlexDistribute", distribute, &distribute, NULL));
4807   PetscCall(PetscOptionsBool("-dm_distribute_save_sf", "Flag to save the migration SF", "DMPlexSetMigrationSF", saveSF, &saveSF, NULL));
4808   PetscCall(DMSetFromOptions_Overlap_Plex(dm, PetscOptionsObject, &overlap));
4809   if (distribute) {
4810     DM               pdm = NULL;
4811     PetscPartitioner part;
4812     PetscSF          sfMigration;
4813 
4814     PetscCall(DMPlexGetPartitioner(dm, &part));
4815     PetscCall(PetscPartitionerSetFromOptions(part));
4816     PetscCall(DMPlexDistribute(dm, overlap, &sfMigration, &pdm));
4817     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
4818     if (saveSF) PetscCall(DMPlexSetMigrationSF(dm, sfMigration));
4819     PetscCall(PetscSFDestroy(&sfMigration));
4820   }
4821 
4822   {
4823     PetscBool useBoxLabel = PETSC_FALSE;
4824     PetscCall(PetscOptionsBool("-dm_plex_box_label", "Create 'Face Sets' assuming boundary faces align with cartesian directions", "DMCreate", useBoxLabel, &useBoxLabel, NULL));
4825     if (useBoxLabel) {
4826       PetscInt       n      = 3;
4827       DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
4828 
4829       PetscCall(PetscOptionsEnumArray("-dm_plex_box_label_bd", "Boundary type for each dimension when using -dm_plex_box_label", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
4830       PetscCheck(!flg || !(n != dim), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4831       PetscCall(DMPlexSetBoxLabel_Internal(dm, bdt));
4832     }
4833   }
4834   /* Must check CEED options before creating function space for coordinates */
4835   {
4836     PetscBool useCeed = PETSC_FALSE, flg;
4837 
4838     PetscCall(PetscOptionsBool("-dm_plex_use_ceed", "Use LibCEED as the FEM backend", "DMPlexSetUseCeed", useCeed, &useCeed, &flg));
4839     if (flg) PetscCall(DMPlexSetUseCeed(dm, useCeed));
4840   }
4841   /* Create coordinate space */
4842   if (created) {
4843     DM_Plex  *mesh   = (DM_Plex *)dm->data;
4844     PetscInt  degree = 1, deg;
4845     PetscInt  height = 0;
4846     DM        cdm;
4847     PetscBool flg, localize = PETSC_TRUE, sparseLocalize = PETSC_TRUE;
4848 
4849     PetscCall(PetscOptionsBool("-dm_coord_space", "Use an FEM space for coordinates", "", coordSpace, &coordSpace, &flg));
4850     PetscCall(PetscOptionsInt("-dm_coord_petscspace_degree", "FEM degree for coordinate space", "", degree, &degree, NULL));
4851     PetscCall(DMGetCoordinateDegree_Internal(dm, &deg));
4852     if (coordSpace && deg <= 1) PetscCall(DMPlexCreateCoordinateSpace(dm, degree, PETSC_TRUE, mesh->coordFunc));
4853     PetscCall(DMGetCoordinateDM(dm, &cdm));
4854     if (flg && !coordSpace) {
4855       PetscDS      cds;
4856       PetscObject  obj;
4857       PetscClassId id;
4858 
4859       PetscCall(DMGetDS(cdm, &cds));
4860       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
4861       PetscCall(PetscObjectGetClassId(obj, &id));
4862       if (id == PETSCFE_CLASSID) {
4863         PetscContainer dummy;
4864 
4865         PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &dummy));
4866         PetscCall(PetscObjectSetName((PetscObject)dummy, "coordinates"));
4867         PetscCall(DMSetField(cdm, 0, NULL, (PetscObject)dummy));
4868         PetscCall(PetscContainerDestroy(&dummy));
4869         PetscCall(DMClearDS(cdm));
4870       }
4871       mesh->coordFunc = NULL;
4872     }
4873     PetscCall(PetscOptionsBool("-dm_localize", "Localize mesh coordinates", "", localize, &localize, NULL));
4874     PetscCall(PetscOptionsBool("-dm_sparse_localize", "Localize only necessary cells", "DMSetSparseLocalize", sparseLocalize, &sparseLocalize, &flg));
4875     if (flg) PetscCall(DMSetSparseLocalize(dm, sparseLocalize));
4876     PetscCall(PetscOptionsInt("-dm_localize_height", "Localize edges and faces in addition to cells", "", height, &height, &flg));
4877     if (flg) PetscCall(DMPlexSetMaxProjectionHeight(cdm, height));
4878     if (localize) PetscCall(DMLocalizeCoordinates(dm));
4879   }
4880   /* Handle DMPlex refinement */
4881   remap = PETSC_TRUE;
4882   PetscCall(PetscOptionsBoundedInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL, 0));
4883   PetscCall(PetscOptionsBool("-dm_refine_remap", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
4884   PetscCall(PetscOptionsBoundedInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy, 0));
4885   if (refine) PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
4886   if (refine && isHierarchy) {
4887     DM *dms, coarseDM;
4888 
4889     PetscCall(DMGetCoarseDM(dm, &coarseDM));
4890     PetscCall(PetscObjectReference((PetscObject)coarseDM));
4891     PetscCall(PetscMalloc1(refine, &dms));
4892     PetscCall(DMRefineHierarchy(dm, refine, dms));
4893     /* Total hack since we do not pass in a pointer */
4894     PetscCall(DMPlexSwap_Static(dm, dms[refine - 1]));
4895     if (refine == 1) {
4896       PetscCall(DMSetCoarseDM(dm, dms[0]));
4897       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4898     } else {
4899       PetscCall(DMSetCoarseDM(dm, dms[refine - 2]));
4900       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
4901       PetscCall(DMSetCoarseDM(dms[0], dms[refine - 1]));
4902       PetscCall(DMPlexSetRegularRefinement(dms[0], PETSC_TRUE));
4903     }
4904     PetscCall(DMSetCoarseDM(dms[refine - 1], coarseDM));
4905     PetscCall(PetscObjectDereference((PetscObject)coarseDM));
4906     /* Free DMs */
4907     for (r = 0; r < refine; ++r) {
4908       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4909       PetscCall(DMDestroy(&dms[r]));
4910     }
4911     PetscCall(PetscFree(dms));
4912   } else {
4913     for (r = 0; r < refine; ++r) {
4914       DM             rdm;
4915       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4916 
4917       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4918       PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
4919       /* Total hack since we do not pass in a pointer */
4920       PetscCall(DMPlexReplace_Internal(dm, &rdm));
4921       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4922       if (coordFunc && remap) {
4923         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4924         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4925       }
4926     }
4927   }
4928   /* Handle DMPlex coarsening */
4929   PetscCall(PetscOptionsBoundedInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL, 0));
4930   PetscCall(PetscOptionsBoundedInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy, 0));
4931   if (coarsen && isHierarchy) {
4932     DM *dms;
4933 
4934     PetscCall(PetscMalloc1(coarsen, &dms));
4935     PetscCall(DMCoarsenHierarchy(dm, coarsen, dms));
4936     /* Free DMs */
4937     for (r = 0; r < coarsen; ++r) {
4938       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
4939       PetscCall(DMDestroy(&dms[r]));
4940     }
4941     PetscCall(PetscFree(dms));
4942   } else {
4943     for (r = 0; r < coarsen; ++r) {
4944       DM             cdm;
4945       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
4946 
4947       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4948       PetscCall(DMCoarsen(dm, PetscObjectComm((PetscObject)dm), &cdm));
4949       /* Total hack since we do not pass in a pointer */
4950       PetscCall(DMPlexReplace_Internal(dm, &cdm));
4951       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
4952       if (coordFunc) {
4953         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
4954         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
4955       }
4956     }
4957   }
4958   // Handle coordinate remapping
4959   remap = PETSC_FALSE;
4960   PetscCall(PetscOptionsBool("-dm_coord_remap", "Flag to control coordinate remapping", "", remap, &remap, NULL));
4961   if (remap) {
4962     DMPlexCoordMap map     = DM_COORD_MAP_NONE;
4963     PetscPointFunc mapFunc = NULL;
4964     PetscScalar    params[16];
4965     PetscInt       Np = PETSC_STATIC_ARRAY_LENGTH(params), cdim;
4966     MPI_Comm       comm;
4967 
4968     PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
4969     PetscCall(DMGetCoordinateDim(dm, &cdim));
4970     PetscCall(PetscOptionsScalarArray("-dm_coord_map_params", "Parameters for the coordinate remapping", "", params, &Np, &flg));
4971     if (!flg) Np = 0;
4972     // TODO Allow user to pass a map function by name
4973     PetscCall(PetscOptionsEnum("-dm_coord_map", "Coordinate mapping for built-in mesh", "", DMPlexCoordMaps, (PetscEnum)map, (PetscEnum *)&map, &flg));
4974     if (flg) {
4975       switch (map) {
4976       case DM_COORD_MAP_NONE:
4977         mapFunc = coordMap_identity;
4978         break;
4979       case DM_COORD_MAP_SHEAR:
4980         mapFunc = coordMap_shear;
4981         if (!Np) {
4982           Np        = cdim + 1;
4983           params[0] = 0;
4984           for (PetscInt d = 1; d <= cdim; ++d) params[d] = 1.0;
4985         }
4986         PetscCheck(Np == cdim + 1, comm, PETSC_ERR_ARG_WRONG, "The shear coordinate map must have cdim + 1 = %" PetscInt_FMT " parameters, not %" PetscInt_FMT, cdim + 1, Np);
4987         break;
4988       case DM_COORD_MAP_FLARE:
4989         mapFunc = coordMap_flare;
4990         if (!Np) {
4991           Np        = cdim + 1;
4992           params[0] = 0;
4993           for (PetscInt d = 1; d <= cdim; ++d) params[d] = 1.0;
4994         }
4995         PetscCheck(Np == cdim + 1, comm, PETSC_ERR_ARG_WRONG, "The flare coordinate map must have cdim + 1 = %" PetscInt_FMT " parameters, not %" PetscInt_FMT, cdim + 1, Np);
4996         break;
4997       case DM_COORD_MAP_ANNULUS:
4998         mapFunc = coordMap_annulus;
4999         if (!Np) {
5000           Np        = 2;
5001           params[0] = 1.;
5002           params[1] = 2.;
5003         }
5004         PetscCheck(Np == 2, comm, PETSC_ERR_ARG_WRONG, "The annulus coordinate map must have 2 parameters, not %" PetscInt_FMT, Np);
5005         break;
5006       case DM_COORD_MAP_SHELL:
5007         mapFunc = coordMap_shell;
5008         if (!Np) {
5009           Np        = 2;
5010           params[0] = 1.;
5011           params[1] = 2.;
5012         }
5013         PetscCheck(Np == 2, comm, PETSC_ERR_ARG_WRONG, "The spherical shell coordinate map must have 2 parameters, not %" PetscInt_FMT, Np);
5014         break;
5015       default:
5016         mapFunc = coordMap_identity;
5017       }
5018     }
5019     if (Np) {
5020       DM      cdm;
5021       PetscDS cds;
5022 
5023       PetscCall(DMGetCoordinateDM(dm, &cdm));
5024       PetscCall(DMGetDS(cdm, &cds));
5025       PetscCall(PetscDSSetConstants(cds, Np, params));
5026     }
5027     PetscCall(DMPlexRemapGeometry(dm, 0.0, mapFunc));
5028   }
5029   /* Handle ghost cells */
5030   PetscCall(PetscOptionsBool("-dm_plex_create_fv_ghost_cells", "Flag to create finite volume ghost cells on the boundary", "DMCreate", ghostCells, &ghostCells, NULL));
5031   if (ghostCells) {
5032     DM   gdm;
5033     char lname[PETSC_MAX_PATH_LEN];
5034 
5035     lname[0] = '\0';
5036     PetscCall(PetscOptionsString("-dm_plex_fv_ghost_cells_label", "Label name for ghost cells boundary", "DMCreate", lname, lname, sizeof(lname), &flg));
5037     PetscCall(DMPlexConstructGhostCells(dm, flg ? lname : NULL, NULL, &gdm));
5038     PetscCall(DMPlexReplace_Internal(dm, &gdm));
5039   }
5040   /* Handle 1D order */
5041   if (reorder != DM_REORDER_DEFAULT_FALSE && dim == 1) {
5042     DM           cdm, rdm;
5043     PetscDS      cds;
5044     PetscObject  obj;
5045     PetscClassId id = PETSC_OBJECT_CLASSID;
5046     IS           perm;
5047     PetscInt     Nf;
5048     PetscBool    distributed;
5049 
5050     PetscCall(DMPlexIsDistributed(dm, &distributed));
5051     PetscCall(DMGetCoordinateDM(dm, &cdm));
5052     PetscCall(DMGetDS(cdm, &cds));
5053     PetscCall(PetscDSGetNumFields(cds, &Nf));
5054     if (Nf) {
5055       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
5056       PetscCall(PetscObjectGetClassId(obj, &id));
5057     }
5058     if (!distributed && id != PETSCFE_CLASSID) {
5059       PetscCall(DMPlexGetOrdering1D(dm, &perm));
5060       PetscCall(DMPlexPermute(dm, perm, &rdm));
5061       PetscCall(DMPlexReplace_Internal(dm, &rdm));
5062       PetscCall(ISDestroy(&perm));
5063     }
5064   }
5065 /* Handle */
5066 non_refine:
5067   PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5068   char    *phases[16];
5069   PetscInt Nphases = 16;
5070   PetscCall(PetscOptionsStringArray("-dm_plex_option_phases", "Option phase prefixes", "DMSetFromOptions", phases, &Nphases, &flg));
5071   PetscOptionsHeadEnd();
5072 
5073   // Phases
5074   if (flg) {
5075     const char *oldPrefix;
5076 
5077     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &oldPrefix));
5078     for (PetscInt ph = 0; ph < Nphases; ++ph) {
5079       PetscCall(PetscObjectAppendOptionsPrefix((PetscObject)dm, phases[ph]));
5080       PetscCall(PetscInfo(dm, "Options phase %s for DM %s\n", phases[ph], dm->hdr.name));
5081       PetscCall(DMSetFromOptions(dm));
5082       PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, oldPrefix));
5083       PetscCall(PetscFree(phases[ph]));
5084     }
5085   }
5086   PetscFunctionReturn(PETSC_SUCCESS);
5087 }
5088 
5089 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm, Vec *vec)
5090 {
5091   PetscFunctionBegin;
5092   PetscCall(DMCreateGlobalVector_Section_Private(dm, vec));
5093   /* PetscCall(VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM)); */
5094   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex));
5095   PetscCall(VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void))VecView_Plex_Native));
5096   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex));
5097   PetscCall(VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void))VecLoad_Plex_Native));
5098   PetscFunctionReturn(PETSC_SUCCESS);
5099 }
5100 
5101 static PetscErrorCode DMCreateLocalVector_Plex(DM dm, Vec *vec)
5102 {
5103   PetscFunctionBegin;
5104   PetscCall(DMCreateLocalVector_Section_Private(dm, vec));
5105   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex_Local));
5106   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex_Local));
5107   PetscFunctionReturn(PETSC_SUCCESS);
5108 }
5109 
5110 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
5111 {
5112   PetscInt depth, d;
5113 
5114   PetscFunctionBegin;
5115   PetscCall(DMPlexGetDepth(dm, &depth));
5116   if (depth == 1) {
5117     PetscCall(DMGetDimension(dm, &d));
5118     if (dim == 0) PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
5119     else if (dim == d) PetscCall(DMPlexGetDepthStratum(dm, 1, pStart, pEnd));
5120     else {
5121       *pStart = 0;
5122       *pEnd   = 0;
5123     }
5124   } else {
5125     PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
5126   }
5127   PetscFunctionReturn(PETSC_SUCCESS);
5128 }
5129 
5130 static PetscErrorCode DMGetNeighbors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
5131 {
5132   PetscSF            sf;
5133   PetscMPIInt        niranks, njranks;
5134   PetscInt           n;
5135   const PetscMPIInt *iranks, *jranks;
5136   DM_Plex           *data = (DM_Plex *)dm->data;
5137 
5138   PetscFunctionBegin;
5139   PetscCall(DMGetPointSF(dm, &sf));
5140   if (!data->neighbors) {
5141     PetscCall(PetscSFSetUp(sf));
5142     PetscCall(PetscSFGetRootRanks(sf, &njranks, &jranks, NULL, NULL, NULL));
5143     PetscCall(PetscSFGetLeafRanks(sf, &niranks, &iranks, NULL, NULL));
5144     PetscCall(PetscMalloc1(njranks + niranks + 1, &data->neighbors));
5145     PetscCall(PetscArraycpy(data->neighbors + 1, jranks, njranks));
5146     PetscCall(PetscArraycpy(data->neighbors + njranks + 1, iranks, niranks));
5147     n = njranks + niranks;
5148     PetscCall(PetscSortRemoveDupsMPIInt(&n, data->neighbors + 1));
5149     /* The following cast should never fail: can't have more neighbors than PETSC_MPI_INT_MAX */
5150     PetscCall(PetscMPIIntCast(n, data->neighbors));
5151   }
5152   if (nranks) *nranks = data->neighbors[0];
5153   if (ranks) {
5154     if (data->neighbors[0]) *ranks = data->neighbors + 1;
5155     else *ranks = NULL;
5156   }
5157   PetscFunctionReturn(PETSC_SUCCESS);
5158 }
5159 
5160 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM, DM, Mat, Vec, Vec);
5161 
5162 static PetscErrorCode DMInitialize_Plex(DM dm)
5163 {
5164   PetscFunctionBegin;
5165   dm->ops->view                      = DMView_Plex;
5166   dm->ops->load                      = DMLoad_Plex;
5167   dm->ops->setfromoptions            = DMSetFromOptions_Plex;
5168   dm->ops->clone                     = DMClone_Plex;
5169   dm->ops->setup                     = DMSetUp_Plex;
5170   dm->ops->createlocalsection        = DMCreateLocalSection_Plex;
5171   dm->ops->createsectionpermutation  = DMCreateSectionPermutation_Plex;
5172   dm->ops->createdefaultconstraints  = DMCreateDefaultConstraints_Plex;
5173   dm->ops->createglobalvector        = DMCreateGlobalVector_Plex;
5174   dm->ops->createlocalvector         = DMCreateLocalVector_Plex;
5175   dm->ops->getlocaltoglobalmapping   = NULL;
5176   dm->ops->createfieldis             = NULL;
5177   dm->ops->createcoordinatedm        = DMCreateCoordinateDM_Plex;
5178   dm->ops->createcoordinatefield     = DMCreateCoordinateField_Plex;
5179   dm->ops->getcoloring               = NULL;
5180   dm->ops->creatematrix              = DMCreateMatrix_Plex;
5181   dm->ops->createinterpolation       = DMCreateInterpolation_Plex;
5182   dm->ops->createmassmatrix          = DMCreateMassMatrix_Plex;
5183   dm->ops->createmassmatrixlumped    = DMCreateMassMatrixLumped_Plex;
5184   dm->ops->createinjection           = DMCreateInjection_Plex;
5185   dm->ops->refine                    = DMRefine_Plex;
5186   dm->ops->coarsen                   = DMCoarsen_Plex;
5187   dm->ops->refinehierarchy           = DMRefineHierarchy_Plex;
5188   dm->ops->coarsenhierarchy          = DMCoarsenHierarchy_Plex;
5189   dm->ops->extrude                   = DMExtrude_Plex;
5190   dm->ops->globaltolocalbegin        = NULL;
5191   dm->ops->globaltolocalend          = NULL;
5192   dm->ops->localtoglobalbegin        = NULL;
5193   dm->ops->localtoglobalend          = NULL;
5194   dm->ops->destroy                   = DMDestroy_Plex;
5195   dm->ops->createsubdm               = DMCreateSubDM_Plex;
5196   dm->ops->createsuperdm             = DMCreateSuperDM_Plex;
5197   dm->ops->getdimpoints              = DMGetDimPoints_Plex;
5198   dm->ops->locatepoints              = DMLocatePoints_Plex;
5199   dm->ops->projectfunctionlocal      = DMProjectFunctionLocal_Plex;
5200   dm->ops->projectfunctionlabellocal = DMProjectFunctionLabelLocal_Plex;
5201   dm->ops->projectfieldlocal         = DMProjectFieldLocal_Plex;
5202   dm->ops->projectfieldlabellocal    = DMProjectFieldLabelLocal_Plex;
5203   dm->ops->projectbdfieldlabellocal  = DMProjectBdFieldLabelLocal_Plex;
5204   dm->ops->computel2diff             = DMComputeL2Diff_Plex;
5205   dm->ops->computel2gradientdiff     = DMComputeL2GradientDiff_Plex;
5206   dm->ops->computel2fielddiff        = DMComputeL2FieldDiff_Plex;
5207   dm->ops->getneighbors              = DMGetNeighbors_Plex;
5208   dm->ops->getlocalboundingbox       = DMGetLocalBoundingBox_Coordinates;
5209   dm->ops->createdomaindecomposition = DMCreateDomainDecomposition_Plex;
5210   dm->ops->createddscatters          = DMCreateDomainDecompositionScatters_Plex;
5211   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertBoundaryValues_C", DMPlexInsertBoundaryValues_Plex));
5212   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertTimeDerivativeBoundaryValues_C", DMPlexInsertTimeDerivativeBoundaryValues_Plex));
5213   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMSetUpGLVisViewer_C", DMSetUpGLVisViewer_Plex));
5214   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMCreateNeumannOverlap_C", DMCreateNeumannOverlap_Plex));
5215   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeGetDefault_C", DMPlexDistributeGetDefault_Plex));
5216   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeSetDefault_C", DMPlexDistributeSetDefault_Plex));
5217   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderGetDefault_C", DMPlexReorderGetDefault_Plex));
5218   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderSetDefault_C", DMPlexReorderSetDefault_Plex));
5219   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionGetDefault_C", DMReorderSectionGetDefault_Plex));
5220   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionSetDefault_C", DMReorderSectionSetDefault_Plex));
5221   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionGetType_C", DMReorderSectionGetType_Plex));
5222   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionSetType_C", DMReorderSectionSetType_Plex));
5223   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMInterpolateSolution_C", DMInterpolateSolution_Plex));
5224   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
5225   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetOverlap_C", DMPlexSetOverlap_Plex));
5226   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetUseCeed_C", DMPlexGetUseCeed_Plex));
5227   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetUseCeed_C", DMPlexSetUseCeed_Plex));
5228   PetscFunctionReturn(PETSC_SUCCESS);
5229 }
5230 
5231 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
5232 {
5233   DM_Plex       *mesh = (DM_Plex *)dm->data;
5234   const PetscSF *face_sfs;
5235   PetscInt       num_face_sfs;
5236 
5237   PetscFunctionBegin;
5238   mesh->refct++;
5239   (*newdm)->data = mesh;
5240   PetscCall(DMPlexGetIsoperiodicFaceSF(dm, &num_face_sfs, &face_sfs));
5241   PetscCall(DMPlexSetIsoperiodicFaceSF(*newdm, num_face_sfs, (PetscSF *)face_sfs));
5242   PetscCall(PetscObjectChangeTypeName((PetscObject)*newdm, DMPLEX));
5243   PetscCall(DMInitialize_Plex(*newdm));
5244   PetscFunctionReturn(PETSC_SUCCESS);
5245 }
5246 
5247 /*MC
5248   DMPLEX = "plex" - A `DM` object that encapsulates an unstructured mesh, or CW Complex, which can be expressed using a Hasse Diagram.
5249                     In the local representation, `Vec`s contain all unknowns in the interior and shared boundary. This is
5250                     specified by a PetscSection object. Ownership in the global representation is determined by
5251                     ownership of the underlying `DMPLEX` points. This is specified by another `PetscSection` object.
5252 
5253   Options Database Keys:
5254 + -dm_refine_pre                     - Refine mesh before distribution
5255 + -dm_refine_uniform_pre             - Choose uniform or generator-based refinement
5256 + -dm_refine_volume_limit_pre        - Cell volume limit after pre-refinement using generator
5257 . -dm_distribute                     - Distribute mesh across processes
5258 . -dm_distribute_overlap             - Number of cells to overlap for distribution
5259 . -dm_refine                         - Refine mesh after distribution
5260 . -dm_localize <bool>                - Whether to localize coordinates for periodic meshes
5261 . -dm_sparse_localize <bool>         - Whether to only localize cells on the periodic boundary
5262 . -dm_plex_hash_location             - Use grid hashing for point location
5263 . -dm_plex_hash_box_faces <n,m,p>    - The number of divisions in each direction of the grid hash
5264 . -dm_plex_partition_balance         - Attempt to evenly divide points on partition boundary between processes
5265 . -dm_plex_remesh_bd                 - Allow changes to the boundary on remeshing
5266 . -dm_plex_max_projection_height     - Maximum mesh point height used to project locally
5267 . -dm_plex_regular_refinement        - Use special nested projection algorithm for regular refinement
5268 . -dm_plex_reorder_section           - Use specialized blocking if available
5269 . -dm_plex_check_all                 - Perform all checks below
5270 . -dm_plex_check_symmetry            - Check that the adjacency information in the mesh is symmetric
5271 . -dm_plex_check_skeleton <celltype> - Check that each cell has the correct number of vertices
5272 . -dm_plex_check_faces <celltype>    - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
5273 . -dm_plex_check_geometry            - Check that cells have positive volume
5274 . -dm_view :mesh.tex:ascii_latex     - View the mesh in LaTeX/TikZ
5275 . -dm_plex_view_scale <num>          - Scale the TikZ
5276 . -dm_plex_print_fem <num>           - View FEM assembly information, such as element vectors and matrices
5277 - -dm_plex_print_fvm <num>           - View FVM assembly information, such as flux updates
5278 
5279   Level: intermediate
5280 
5281 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`, `PetscSection`
5282 M*/
5283 
5284 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
5285 {
5286   DM_Plex *mesh;
5287   PetscInt unit;
5288 
5289   PetscFunctionBegin;
5290   PetscCall(PetscCitationsRegister(PlexCitation, &Plexcite));
5291   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5292   PetscCall(PetscNew(&mesh));
5293   dm->reorderSection = DM_REORDER_DEFAULT_NOTSET;
5294   dm->data           = mesh;
5295 
5296   mesh->refct = 1;
5297   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection));
5298   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection));
5299   mesh->refinementUniform      = PETSC_TRUE;
5300   mesh->refinementLimit        = -1.0;
5301   mesh->distDefault            = PETSC_TRUE;
5302   mesh->reorderDefault         = DM_REORDER_DEFAULT_NOTSET;
5303   mesh->distributionName       = NULL;
5304   mesh->interpolated           = DMPLEX_INTERPOLATED_INVALID;
5305   mesh->interpolatedCollective = DMPLEX_INTERPOLATED_INVALID;
5306 
5307   PetscCall(PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner));
5308   mesh->remeshBd = PETSC_FALSE;
5309 
5310   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
5311 
5312   mesh->depthState    = -1;
5313   mesh->celltypeState = -1;
5314   mesh->printTol      = 1.0e-10;
5315   mesh->nonempty_comm = MPI_COMM_SELF;
5316 
5317   PetscCall(DMInitialize_Plex(dm));
5318   PetscFunctionReturn(PETSC_SUCCESS);
5319 }
5320 
5321 /*@
5322   DMPlexCreate - Creates a `DMPLEX` object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
5323 
5324   Collective
5325 
5326   Input Parameter:
5327 . comm - The communicator for the `DMPLEX` object
5328 
5329   Output Parameter:
5330 . mesh - The `DMPLEX` object
5331 
5332   Level: beginner
5333 
5334 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMType`, `DMCreate()`, `DMSetType()`
5335 @*/
5336 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
5337 {
5338   PetscFunctionBegin;
5339   PetscAssertPointer(mesh, 2);
5340   PetscCall(DMCreate(comm, mesh));
5341   PetscCall(DMSetType(*mesh, DMPLEX));
5342   PetscFunctionReturn(PETSC_SUCCESS);
5343 }
5344 
5345 /*@C
5346   DMPlexBuildFromCellListParallel - Build distributed `DMPLEX` topology from a list of vertices for each cell (common mesh generator output) where all cells have the same celltype
5347 
5348   Collective; No Fortran Support
5349 
5350   Input Parameters:
5351 + dm          - The `DM`
5352 . numCells    - The number of cells owned by this process
5353 . numVertices - The number of vertices to be owned by this process, or `PETSC_DECIDE`
5354 . NVertices   - The global number of vertices, or `PETSC_DETERMINE`
5355 . numCorners  - The number of vertices for each cell
5356 - cells       - An array of numCells*numCorners numbers, the global vertex numbers for each cell
5357 
5358   Output Parameters:
5359 + vertexSF         - (Optional) `PetscSF` describing complete vertex ownership
5360 - verticesAdjSaved - (Optional) vertex adjacency array
5361 
5362   Level: advanced
5363 
5364   Notes:
5365   Two triangles sharing a face
5366 .vb
5367 
5368         2
5369       / | \
5370      /  |  \
5371     /   |   \
5372    0  0 | 1  3
5373     \   |   /
5374      \  |  /
5375       \ | /
5376         1
5377 .ve
5378   would have input
5379 .vb
5380   numCells = 2, numVertices = 4
5381   cells = [0 1 2  1 3 2]
5382 .ve
5383   which would result in the `DMPLEX`
5384 .vb
5385 
5386         4
5387       / | \
5388      /  |  \
5389     /   |   \
5390    2  0 | 1  5
5391     \   |   /
5392      \  |  /
5393       \ | /
5394         3
5395 .ve
5396 
5397   Vertices are implicitly numbered consecutively 0,...,NVertices.
5398   Each rank owns a chunk of numVertices consecutive vertices.
5399   If numVertices is `PETSC_DECIDE`, PETSc will distribute them as evenly as possible using PetscLayout.
5400   If NVertices is `PETSC_DETERMINE` and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
5401   If only NVertices is `PETSC_DETERMINE`, it is computed as the sum of numVertices over all ranks.
5402 
5403   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
5404 
5405 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildCoordinatesFromCellListParallel()`,
5406           `PetscSF`
5407 @*/
5408 PetscErrorCode DMPlexBuildFromCellListParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, const PetscInt cells[], PetscSF *vertexSF, PetscInt **verticesAdjSaved)
5409 {
5410   PetscSF     sfPoint;
5411   PetscLayout layout;
5412   PetscInt    numVerticesAdj, *verticesAdj, *cones, c, p;
5413 
5414   PetscFunctionBegin;
5415   PetscValidLogicalCollectiveInt(dm, NVertices, 4);
5416   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5417   /* Get/check global number of vertices */
5418   {
5419     PetscInt       NVerticesInCells, i;
5420     const PetscInt len = numCells * numCorners;
5421 
5422     /* NVerticesInCells = max(cells) + 1 */
5423     NVerticesInCells = PETSC_INT_MIN;
5424     for (i = 0; i < len; i++)
5425       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
5426     ++NVerticesInCells;
5427     PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
5428 
5429     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
5430     else
5431       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
5432   }
5433   /* Count locally unique vertices */
5434   {
5435     PetscHSetI vhash;
5436     PetscInt   off = 0;
5437 
5438     PetscCall(PetscHSetICreate(&vhash));
5439     for (c = 0; c < numCells; ++c) {
5440       for (p = 0; p < numCorners; ++p) PetscCall(PetscHSetIAdd(vhash, cells[c * numCorners + p]));
5441     }
5442     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
5443     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
5444     else verticesAdj = *verticesAdjSaved;
5445     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
5446     PetscCall(PetscHSetIDestroy(&vhash));
5447     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
5448   }
5449   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
5450   /* Create cones */
5451   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
5452   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
5453   PetscCall(DMSetUp(dm));
5454   PetscCall(DMPlexGetCones(dm, &cones));
5455   for (c = 0; c < numCells; ++c) {
5456     for (p = 0; p < numCorners; ++p) {
5457       const PetscInt gv = cells[c * numCorners + p];
5458       PetscInt       lv;
5459 
5460       /* Positions within verticesAdj form 0-based local vertex numbering;
5461          we need to shift it by numCells to get correct DAG points (cells go first) */
5462       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
5463       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
5464       cones[c * numCorners + p] = lv + numCells;
5465     }
5466   }
5467   /* Build point sf */
5468   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
5469   PetscCall(PetscLayoutSetSize(layout, NVertices));
5470   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
5471   PetscCall(PetscLayoutSetBlockSize(layout, 1));
5472   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
5473   PetscCall(PetscLayoutDestroy(&layout));
5474   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
5475   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
5476   if (dm->sf) {
5477     const char *prefix;
5478 
5479     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
5480     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
5481   }
5482   PetscCall(DMSetPointSF(dm, sfPoint));
5483   PetscCall(PetscSFDestroy(&sfPoint));
5484   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)*vertexSF, "Vertex Ownership SF"));
5485   /* Fill in the rest of the topology structure */
5486   PetscCall(DMPlexSymmetrize(dm));
5487   PetscCall(DMPlexStratify(dm));
5488   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5489   PetscFunctionReturn(PETSC_SUCCESS);
5490 }
5491 
5492 /*@C
5493   DMPlexBuildFromCellSectionParallel - Build distributed `DMPLEX` topology from a list of vertices for each cell (common mesh generator output) allowing multiple celltypes
5494 
5495   Collective; No Fortran Support
5496 
5497   Input Parameters:
5498 + dm          - The `DM`
5499 . numCells    - The number of cells owned by this process
5500 . numVertices - The number of vertices to be owned by this process, or `PETSC_DECIDE`
5501 . NVertices   - The global number of vertices, or `PETSC_DETERMINE`
5502 . cellSection - The `PetscSection` giving the number of vertices for each cell (layout of cells)
5503 - cells       - An array of the global vertex numbers for each cell
5504 
5505   Output Parameters:
5506 + vertexSF         - (Optional) `PetscSF` describing complete vertex ownership
5507 - verticesAdjSaved - (Optional) vertex adjacency array
5508 
5509   Level: advanced
5510 
5511   Notes:
5512   A triangle and quadrilateral sharing a face
5513 .vb
5514         2----------3
5515       / |          |
5516      /  |          |
5517     /   |          |
5518    0  0 |     1    |
5519     \   |          |
5520      \  |          |
5521       \ |          |
5522         1----------4
5523 .ve
5524   would have input
5525 .vb
5526   numCells = 2, numVertices = 5
5527   cells = [0 1 2  1 4 3 2]
5528 .ve
5529   which would result in the `DMPLEX`
5530 .vb
5531         4----------5
5532       / |          |
5533      /  |          |
5534     /   |          |
5535    2  0 |     1    |
5536     \   |          |
5537      \  |          |
5538       \ |          |
5539         3----------6
5540 .ve
5541 
5542   Vertices are implicitly numbered consecutively 0,...,NVertices.
5543   Each rank owns a chunk of numVertices consecutive vertices.
5544   If numVertices is `PETSC_DECIDE`, PETSc will distribute them as evenly as possible using PetscLayout.
5545   If NVertices is `PETSC_DETERMINE` and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
5546   If only NVertices is `PETSC_DETERMINE`, it is computed as the sum of numVertices over all ranks.
5547 
5548   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
5549 
5550 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellListParallel()`, `DMPlexCreateFromCellSectionParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`,
5551           `PetscSF`
5552 @*/
5553 PetscErrorCode DMPlexBuildFromCellSectionParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscSection cellSection, const PetscInt cells[], PetscSF *vertexSF, PetscInt **verticesAdjSaved)
5554 {
5555   PetscSF     sfPoint;
5556   PetscLayout layout;
5557   PetscInt    numVerticesAdj, *verticesAdj, *cones, cStart, cEnd, len;
5558 
5559   PetscFunctionBegin;
5560   PetscValidLogicalCollectiveInt(dm, NVertices, 4);
5561   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5562   PetscCall(PetscSectionGetChart(cellSection, &cStart, &cEnd));
5563   PetscCall(PetscSectionGetStorageSize(cellSection, &len));
5564   PetscCheck(cStart == 0 && cEnd == numCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Section chart [%" PetscInt_FMT ", %" PetscInt_FMT ") should be [0, %" PetscInt_FMT ")", cStart, cEnd, numCells);
5565   /* Get/check global number of vertices */
5566   {
5567     PetscInt NVerticesInCells;
5568 
5569     /* NVerticesInCells = max(cells) + 1 */
5570     NVerticesInCells = PETSC_MIN_INT;
5571     for (PetscInt i = 0; i < len; i++)
5572       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
5573     ++NVerticesInCells;
5574     PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
5575 
5576     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
5577     else
5578       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
5579   }
5580   /* Count locally unique vertices */
5581   {
5582     PetscHSetI vhash;
5583     PetscInt   off = 0;
5584 
5585     PetscCall(PetscHSetICreate(&vhash));
5586     for (PetscInt i = 0; i < len; i++) PetscCall(PetscHSetIAdd(vhash, cells[i]));
5587     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
5588     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
5589     else verticesAdj = *verticesAdjSaved;
5590     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
5591     PetscCall(PetscHSetIDestroy(&vhash));
5592     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
5593   }
5594   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
5595   /* Create cones */
5596   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
5597   for (PetscInt c = 0; c < numCells; ++c) {
5598     PetscInt dof;
5599 
5600     PetscCall(PetscSectionGetDof(cellSection, c, &dof));
5601     PetscCall(DMPlexSetConeSize(dm, c, dof));
5602   }
5603   PetscCall(DMSetUp(dm));
5604   PetscCall(DMPlexGetCones(dm, &cones));
5605   for (PetscInt c = 0; c < numCells; ++c) {
5606     PetscInt dof, off;
5607 
5608     PetscCall(PetscSectionGetDof(cellSection, c, &dof));
5609     PetscCall(PetscSectionGetOffset(cellSection, c, &off));
5610     for (PetscInt p = off; p < off + dof; ++p) {
5611       const PetscInt gv = cells[p];
5612       PetscInt       lv;
5613 
5614       /* Positions within verticesAdj form 0-based local vertex numbering;
5615          we need to shift it by numCells to get correct DAG points (cells go first) */
5616       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
5617       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
5618       cones[p] = lv + numCells;
5619     }
5620   }
5621   /* Build point sf */
5622   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
5623   PetscCall(PetscLayoutSetSize(layout, NVertices));
5624   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
5625   PetscCall(PetscLayoutSetBlockSize(layout, 1));
5626   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
5627   PetscCall(PetscLayoutDestroy(&layout));
5628   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
5629   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
5630   if (dm->sf) {
5631     const char *prefix;
5632 
5633     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
5634     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
5635   }
5636   PetscCall(DMSetPointSF(dm, sfPoint));
5637   PetscCall(PetscSFDestroy(&sfPoint));
5638   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)*vertexSF, "Vertex Ownership SF"));
5639   /* Fill in the rest of the topology structure */
5640   PetscCall(DMPlexSymmetrize(dm));
5641   PetscCall(DMPlexStratify(dm));
5642   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5643   PetscFunctionReturn(PETSC_SUCCESS);
5644 }
5645 
5646 /*@
5647   DMPlexBuildCoordinatesFromCellListParallel - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
5648 
5649   Collective; No Fortran Support
5650 
5651   Input Parameters:
5652 + dm           - The `DM`
5653 . spaceDim     - The spatial dimension used for coordinates
5654 . sfVert       - `PetscSF` describing complete vertex ownership
5655 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
5656 
5657   Level: advanced
5658 
5659 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellListParallel()`
5660 @*/
5661 PetscErrorCode DMPlexBuildCoordinatesFromCellListParallel(DM dm, PetscInt spaceDim, PetscSF sfVert, const PetscReal vertexCoords[])
5662 {
5663   PetscSection coordSection;
5664   Vec          coordinates;
5665   PetscScalar *coords;
5666   PetscInt     numVertices, numVerticesAdj, coordSize, v, vStart, vEnd;
5667   PetscMPIInt  spaceDimi;
5668 
5669   PetscFunctionBegin;
5670   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
5671   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
5672   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
5673   PetscCall(DMSetCoordinateDim(dm, spaceDim));
5674   PetscCall(PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL));
5675   PetscCheck(vEnd - vStart == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Supplied sfVert has wrong number of leaves = %" PetscInt_FMT " != %" PetscInt_FMT " = vEnd - vStart", numVerticesAdj, vEnd - vStart);
5676   PetscCall(DMGetCoordinateSection(dm, &coordSection));
5677   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5678   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
5679   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
5680   for (v = vStart; v < vEnd; ++v) {
5681     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
5682     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
5683   }
5684   PetscCall(PetscSectionSetUp(coordSection));
5685   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
5686   PetscCall(VecCreate(PetscObjectComm((PetscObject)dm), &coordinates));
5687   PetscCall(VecSetBlockSize(coordinates, spaceDim));
5688   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5689   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
5690   PetscCall(VecSetType(coordinates, VECSTANDARD));
5691   PetscCall(VecGetArray(coordinates, &coords));
5692   {
5693     MPI_Datatype coordtype;
5694 
5695     /* Need a temp buffer for coords if we have complex/single */
5696     PetscCall(PetscMPIIntCast(spaceDim, &spaceDimi));
5697     PetscCallMPI(MPI_Type_contiguous(spaceDimi, MPIU_SCALAR, &coordtype));
5698     PetscCallMPI(MPI_Type_commit(&coordtype));
5699 #if defined(PETSC_USE_COMPLEX)
5700     {
5701       PetscScalar *svertexCoords;
5702       PetscInt     i;
5703       PetscCall(PetscMalloc1(numVertices * spaceDim, &svertexCoords));
5704       for (i = 0; i < numVertices * spaceDim; i++) svertexCoords[i] = vertexCoords[i];
5705       PetscCall(PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
5706       PetscCall(PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
5707       PetscCall(PetscFree(svertexCoords));
5708     }
5709 #else
5710     PetscCall(PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
5711     PetscCall(PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
5712 #endif
5713     PetscCallMPI(MPI_Type_free(&coordtype));
5714   }
5715   PetscCall(VecRestoreArray(coordinates, &coords));
5716   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
5717   PetscCall(VecDestroy(&coordinates));
5718   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
5719   PetscFunctionReturn(PETSC_SUCCESS);
5720 }
5721 
5722 /*@
5723   DMPlexCreateFromCellListParallelPetsc - Create distributed `DMPLEX` from a list of vertices for each cell (common mesh generator output) where all cells have the same celltype
5724 
5725   Collective
5726 
5727   Input Parameters:
5728 + comm         - The communicator
5729 . dim          - The topological dimension of the mesh
5730 . numCells     - The number of cells owned by this process
5731 . numVertices  - The number of vertices owned by this process, or `PETSC_DECIDE`
5732 . NVertices    - The global number of vertices, or `PETSC_DECIDE`
5733 . numCorners   - The number of vertices for each cell
5734 . interpolate  - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
5735 . cells        - An array of numCells*numCorners numbers, the global vertex numbers for each cell
5736 . spaceDim     - The spatial dimension used for coordinates
5737 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
5738 
5739   Output Parameters:
5740 + dm          - The `DM`
5741 . vertexSF    - (Optional) `PetscSF` describing complete vertex ownership
5742 - verticesAdj - (Optional) vertex adjacency array
5743 
5744   Level: intermediate
5745 
5746   Notes:
5747   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`,
5748   `DMPlexBuildFromCellListParallel()`, `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellListParallel()`
5749 
5750   See `DMPlexBuildFromCellListParallel()` for an example and details about the topology-related parameters.
5751 
5752   See `DMPlexBuildCoordinatesFromCellListParallel()` for details about the geometry-related parameters.
5753 
5754 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
5755 @*/
5756 PetscErrorCode DMPlexCreateFromCellListParallelPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, PetscInt **verticesAdj, DM *dm)
5757 {
5758   PetscSF sfVert;
5759 
5760   PetscFunctionBegin;
5761   PetscCall(DMCreate(comm, dm));
5762   PetscCall(DMSetType(*dm, DMPLEX));
5763   PetscValidLogicalCollectiveInt(*dm, dim, 2);
5764   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
5765   PetscCall(DMSetDimension(*dm, dim));
5766   PetscCall(DMPlexBuildFromCellListParallel(*dm, numCells, numVertices, NVertices, numCorners, cells, &sfVert, verticesAdj));
5767   if (interpolate) {
5768     DM idm;
5769 
5770     PetscCall(DMPlexInterpolate(*dm, &idm));
5771     PetscCall(DMDestroy(dm));
5772     *dm = idm;
5773   }
5774   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
5775   if (vertexSF) *vertexSF = sfVert;
5776   else PetscCall(PetscSFDestroy(&sfVert));
5777   PetscFunctionReturn(PETSC_SUCCESS);
5778 }
5779 
5780 /*@
5781   DMPlexCreateFromCellSectionParallel - Create distributed `DMPLEX` from a list of vertices for each cell (common mesh generator output) and supports multiple celltypes
5782 
5783   Collective
5784 
5785   Input Parameters:
5786 + comm         - The communicator
5787 . dim          - The topological dimension of the mesh
5788 . numCells     - The number of cells owned by this process
5789 . numVertices  - The number of vertices owned by this process, or `PETSC_DECIDE`
5790 . NVertices    - The global number of vertices, or `PETSC_DECIDE`
5791 . cellSection  - The `PetscSection` giving the number of vertices for each cell (layout of cells)
5792 . interpolate  - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
5793 . cells        - An array of the global vertex numbers for each cell
5794 . spaceDim     - The spatial dimension used for coordinates
5795 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
5796 
5797   Output Parameters:
5798 + dm          - The `DM`
5799 . vertexSF    - (Optional) `PetscSF` describing complete vertex ownership
5800 - verticesAdj - (Optional) vertex adjacency array
5801 
5802   Level: intermediate
5803 
5804   Notes:
5805   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`,
5806   `DMPlexBuildFromCellSectionParallel()`, `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellListParallel()`
5807 
5808   See `DMPlexBuildFromCellSectionParallel()` for an example and details about the topology-related parameters.
5809 
5810   See `DMPlexBuildCoordinatesFromCellListParallel()` for details about the geometry-related parameters.
5811 
5812 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
5813 @*/
5814 PetscErrorCode DMPlexCreateFromCellSectionParallel(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscSection cellSection, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PetscSF *vertexSF, PetscInt **verticesAdj, DM *dm)
5815 {
5816   PetscSF sfVert;
5817 
5818   PetscFunctionBegin;
5819   PetscCall(DMCreate(comm, dm));
5820   PetscCall(DMSetType(*dm, DMPLEX));
5821   PetscValidLogicalCollectiveInt(*dm, dim, 2);
5822   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
5823   PetscCall(DMSetDimension(*dm, dim));
5824   PetscCall(DMPlexBuildFromCellSectionParallel(*dm, numCells, numVertices, NVertices, cellSection, cells, &sfVert, verticesAdj));
5825   if (interpolate) {
5826     DM idm;
5827 
5828     PetscCall(DMPlexInterpolate(*dm, &idm));
5829     PetscCall(DMDestroy(dm));
5830     *dm = idm;
5831   }
5832   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
5833   if (vertexSF) *vertexSF = sfVert;
5834   else PetscCall(PetscSFDestroy(&sfVert));
5835   PetscFunctionReturn(PETSC_SUCCESS);
5836 }
5837 
5838 /*@
5839   DMPlexBuildFromCellList - Build `DMPLEX` topology from a list of vertices for each cell (common mesh generator output)
5840 
5841   Collective; No Fortran Support
5842 
5843   Input Parameters:
5844 + dm          - The `DM`
5845 . numCells    - The number of cells owned by this process
5846 . numVertices - The number of vertices owned by this process, or `PETSC_DETERMINE`
5847 . numCorners  - The number of vertices for each cell
5848 - cells       - An array of `numCells` x `numCorners` numbers, the global vertex numbers for each cell
5849 
5850   Level: advanced
5851 
5852   Notes:
5853   Two triangles sharing a face
5854 .vb
5855 
5856         2
5857       / | \
5858      /  |  \
5859     /   |   \
5860    0  0 | 1  3
5861     \   |   /
5862      \  |  /
5863       \ | /
5864         1
5865 .ve
5866   would have input
5867 .vb
5868   numCells = 2, numVertices = 4
5869   cells = [0 1 2  1 3 2]
5870 .ve
5871   which would result in the `DMPLEX`
5872 .vb
5873 
5874         4
5875       / | \
5876      /  |  \
5877     /   |   \
5878    2  0 | 1  5
5879     \   |   /
5880      \  |  /
5881       \ | /
5882         3
5883 .ve
5884 
5885   If numVertices is `PETSC_DETERMINE`, it is computed by PETSc as the maximum vertex index in cells + 1.
5886 
5887 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListPetsc()`
5888 @*/
5889 PetscErrorCode DMPlexBuildFromCellList(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const PetscInt cells[])
5890 {
5891   PetscInt *cones, c, p, dim;
5892 
5893   PetscFunctionBegin;
5894   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5895   PetscCall(DMGetDimension(dm, &dim));
5896   /* Get/check global number of vertices */
5897   {
5898     PetscInt       NVerticesInCells, i;
5899     const PetscInt len = numCells * numCorners;
5900 
5901     /* NVerticesInCells = max(cells) + 1 */
5902     NVerticesInCells = PETSC_INT_MIN;
5903     for (i = 0; i < len; i++)
5904       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
5905     ++NVerticesInCells;
5906 
5907     if (numVertices == PETSC_DECIDE) numVertices = NVerticesInCells;
5908     else
5909       PetscCheck(numVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, numVertices, NVerticesInCells);
5910   }
5911   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
5912   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
5913   PetscCall(DMSetUp(dm));
5914   PetscCall(DMPlexGetCones(dm, &cones));
5915   for (c = 0; c < numCells; ++c) {
5916     for (p = 0; p < numCorners; ++p) cones[c * numCorners + p] = cells[c * numCorners + p] + numCells;
5917   }
5918   PetscCall(DMPlexSymmetrize(dm));
5919   PetscCall(DMPlexStratify(dm));
5920   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5921   PetscFunctionReturn(PETSC_SUCCESS);
5922 }
5923 
5924 /*@
5925   DMPlexBuildCoordinatesFromCellList - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
5926 
5927   Collective
5928 
5929   Input Parameters:
5930 + dm           - The `DM`
5931 . spaceDim     - The spatial dimension used for coordinates
5932 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
5933 
5934   Level: advanced
5935 
5936 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellList()`
5937 @*/
5938 PetscErrorCode DMPlexBuildCoordinatesFromCellList(DM dm, PetscInt spaceDim, const PetscReal vertexCoords[])
5939 {
5940   PetscSection coordSection;
5941   Vec          coordinates;
5942   DM           cdm;
5943   PetscScalar *coords;
5944   PetscInt     v, vStart, vEnd, d;
5945 
5946   PetscFunctionBegin;
5947   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
5948   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
5949   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
5950   PetscCall(DMSetCoordinateDim(dm, spaceDim));
5951   PetscCall(DMGetCoordinateSection(dm, &coordSection));
5952   PetscCall(PetscSectionSetNumFields(coordSection, 1));
5953   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
5954   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
5955   for (v = vStart; v < vEnd; ++v) {
5956     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
5957     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
5958   }
5959   PetscCall(PetscSectionSetUp(coordSection));
5960 
5961   PetscCall(DMGetCoordinateDM(dm, &cdm));
5962   PetscCall(DMCreateLocalVector(cdm, &coordinates));
5963   PetscCall(VecSetBlockSize(coordinates, spaceDim));
5964   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
5965   PetscCall(VecGetArrayWrite(coordinates, &coords));
5966   for (v = 0; v < vEnd - vStart; ++v) {
5967     for (d = 0; d < spaceDim; ++d) coords[v * spaceDim + d] = vertexCoords[v * spaceDim + d];
5968   }
5969   PetscCall(VecRestoreArrayWrite(coordinates, &coords));
5970   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
5971   PetscCall(VecDestroy(&coordinates));
5972   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
5973   PetscFunctionReturn(PETSC_SUCCESS);
5974 }
5975 
5976 /*@
5977   DMPlexCreateFromCellListPetsc - Create `DMPLEX` from a list of vertices for each cell (common mesh generator output), but only process 0 takes in the input
5978 
5979   Collective
5980 
5981   Input Parameters:
5982 + comm         - The communicator
5983 . dim          - The topological dimension of the mesh
5984 . numCells     - The number of cells, only on process 0
5985 . numVertices  - The number of vertices owned by this process, or `PETSC_DECIDE`, only on process 0
5986 . numCorners   - The number of vertices for each cell, only on process 0
5987 . interpolate  - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
5988 . cells        - An array of numCells*numCorners numbers, the vertices for each cell, only on process 0
5989 . spaceDim     - The spatial dimension used for coordinates
5990 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex, only on process 0
5991 
5992   Output Parameter:
5993 . dm - The `DM`, which only has points on process 0
5994 
5995   Level: intermediate
5996 
5997   Notes:
5998   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`, `DMPlexBuildFromCellList()`,
5999   `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellList()`
6000 
6001   See `DMPlexBuildFromCellList()` for an example and details about the topology-related parameters.
6002   See `DMPlexBuildCoordinatesFromCellList()` for details about the geometry-related parameters.
6003   See `DMPlexCreateFromCellListParallelPetsc()` for parallel input
6004 
6005 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellList()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
6006 @*/
6007 PetscErrorCode DMPlexCreateFromCellListPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], DM *dm)
6008 {
6009   PetscMPIInt rank;
6010 
6011   PetscFunctionBegin;
6012   PetscCheck(dim, comm, PETSC_ERR_ARG_OUTOFRANGE, "This is not appropriate for 0-dimensional meshes. Consider either creating the DM using DMPlexCreateFromDAG(), by hand, or using DMSwarm.");
6013   PetscCallMPI(MPI_Comm_rank(comm, &rank));
6014   PetscCall(DMCreate(comm, dm));
6015   PetscCall(DMSetType(*dm, DMPLEX));
6016   PetscCall(DMSetDimension(*dm, dim));
6017   if (rank == 0) PetscCall(DMPlexBuildFromCellList(*dm, numCells, numVertices, numCorners, cells));
6018   else PetscCall(DMPlexBuildFromCellList(*dm, 0, 0, 0, NULL));
6019   if (interpolate) {
6020     DM idm;
6021 
6022     PetscCall(DMPlexInterpolate(*dm, &idm));
6023     PetscCall(DMDestroy(dm));
6024     *dm = idm;
6025   }
6026   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, vertexCoords));
6027   else PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, NULL));
6028   PetscFunctionReturn(PETSC_SUCCESS);
6029 }
6030 
6031 /*@
6032   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a `DM`
6033 
6034   Input Parameters:
6035 + dm               - The empty `DM` object, usually from `DMCreate()` and `DMSetDimension()`
6036 . depth            - The depth of the DAG
6037 . numPoints        - Array of size depth + 1 containing the number of points at each `depth`
6038 . coneSize         - The cone size of each point
6039 . cones            - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
6040 . coneOrientations - The orientation of each cone point
6041 - vertexCoords     - An array of `numPoints`[0]*spacedim numbers representing the coordinates of each vertex, with spacedim the value set via `DMSetCoordinateDim()`
6042 
6043   Output Parameter:
6044 . dm - The `DM`
6045 
6046   Level: advanced
6047 
6048   Note:
6049   Two triangles sharing a face would have input
6050 .vb
6051   depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
6052   cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
6053  vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
6054 .ve
6055   which would result in the DMPlex
6056 .vb
6057         4
6058       / | \
6059      /  |  \
6060     /   |   \
6061    2  0 | 1  5
6062     \   |   /
6063      \  |  /
6064       \ | /
6065         3
6066 .ve
6067   Notice that all points are numbered consecutively, unlike `DMPlexCreateFromCellListPetsc()`
6068 
6069 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
6070 @*/
6071 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
6072 {
6073   Vec          coordinates;
6074   PetscSection coordSection;
6075   PetscScalar *coords;
6076   PetscInt     coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
6077 
6078   PetscFunctionBegin;
6079   PetscCall(DMGetDimension(dm, &dim));
6080   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
6081   PetscCheck(dimEmbed >= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Embedding dimension %" PetscInt_FMT " cannot be less than intrinsic dimension %" PetscInt_FMT, dimEmbed, dim);
6082   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
6083   PetscCall(DMPlexSetChart(dm, pStart, pEnd));
6084   for (p = pStart; p < pEnd; ++p) {
6085     PetscCall(DMPlexSetConeSize(dm, p, coneSize[p - pStart]));
6086     if (firstVertex < 0 && !coneSize[p - pStart]) firstVertex = p - pStart;
6087   }
6088   PetscCheck(firstVertex >= 0 || !numPoints[0], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Expected %" PetscInt_FMT " vertices but could not find any", numPoints[0]);
6089   PetscCall(DMSetUp(dm)); /* Allocate space for cones */
6090   for (p = pStart, off = 0; p < pEnd; off += coneSize[p - pStart], ++p) {
6091     PetscCall(DMPlexSetCone(dm, p, &cones[off]));
6092     PetscCall(DMPlexSetConeOrientation(dm, p, &coneOrientations[off]));
6093   }
6094   PetscCall(DMPlexSymmetrize(dm));
6095   PetscCall(DMPlexStratify(dm));
6096   /* Build coordinates */
6097   PetscCall(DMGetCoordinateSection(dm, &coordSection));
6098   PetscCall(PetscSectionSetNumFields(coordSection, 1));
6099   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dimEmbed));
6100   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numPoints[0]));
6101   for (v = firstVertex; v < firstVertex + numPoints[0]; ++v) {
6102     PetscCall(PetscSectionSetDof(coordSection, v, dimEmbed));
6103     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed));
6104   }
6105   PetscCall(PetscSectionSetUp(coordSection));
6106   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
6107   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
6108   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
6109   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
6110   PetscCall(VecSetBlockSize(coordinates, dimEmbed));
6111   PetscCall(VecSetType(coordinates, VECSTANDARD));
6112   if (vertexCoords) {
6113     PetscCall(VecGetArray(coordinates, &coords));
6114     for (v = 0; v < numPoints[0]; ++v) {
6115       PetscInt off;
6116 
6117       PetscCall(PetscSectionGetOffset(coordSection, v + firstVertex, &off));
6118       for (d = 0; d < dimEmbed; ++d) coords[off + d] = vertexCoords[v * dimEmbed + d];
6119     }
6120   }
6121   PetscCall(VecRestoreArray(coordinates, &coords));
6122   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
6123   PetscCall(VecDestroy(&coordinates));
6124   PetscFunctionReturn(PETSC_SUCCESS);
6125 }
6126 
6127 /*
6128   DMPlexCreateCellVertexFromFile - Create a `DMPLEX` mesh from a simple cell-vertex file.
6129 
6130   Collective
6131 
6132 + comm        - The MPI communicator
6133 . filename    - Name of the .dat file
6134 - interpolate - Create faces and edges in the mesh
6135 
6136   Output Parameter:
6137 . dm  - The `DM` object representing the mesh
6138 
6139   Level: beginner
6140 
6141   Note:
6142   The format is the simplest possible:
6143 .vb
6144   dim Ne Nv Nc Nl
6145   v_1 v_2 ... v_Nc
6146   ...
6147   x y z marker_1 ... marker_Nl
6148 .ve
6149 
6150   Developer Note:
6151   Should use a `PetscViewer` not a filename
6152 
6153 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromFile()`, `DMPlexCreateGmsh()`, `DMPlexCreate()`
6154 */
6155 static PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
6156 {
6157   DMLabel      marker;
6158   PetscViewer  viewer;
6159   Vec          coordinates;
6160   PetscSection coordSection;
6161   PetscScalar *coords;
6162   char         line[PETSC_MAX_PATH_LEN];
6163   PetscInt     cdim, coordSize, v, c, d;
6164   PetscMPIInt  rank;
6165   int          snum, dim, Nv, Nc, Ncn, Nl;
6166 
6167   PetscFunctionBegin;
6168   PetscCallMPI(MPI_Comm_rank(comm, &rank));
6169   PetscCall(PetscViewerCreate(comm, &viewer));
6170   PetscCall(PetscViewerSetType(viewer, PETSCVIEWERASCII));
6171   PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
6172   PetscCall(PetscViewerFileSetName(viewer, filename));
6173   if (rank == 0) {
6174     PetscCall(PetscViewerRead(viewer, line, 5, NULL, PETSC_STRING));
6175     snum = sscanf(line, "%d %d %d %d %d", &dim, &Nc, &Nv, &Ncn, &Nl);
6176     PetscCheck(snum == 5, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
6177   } else {
6178     Nc = Nv = Ncn = Nl = 0;
6179   }
6180   PetscCallMPI(MPI_Bcast(&dim, 1, MPI_INT, 0, comm));
6181   cdim = dim;
6182   PetscCall(DMCreate(comm, dm));
6183   PetscCall(DMSetType(*dm, DMPLEX));
6184   PetscCall(DMPlexSetChart(*dm, 0, Nc + Nv));
6185   PetscCall(DMSetDimension(*dm, dim));
6186   PetscCall(DMSetCoordinateDim(*dm, cdim));
6187   /* Read topology */
6188   if (rank == 0) {
6189     char     format[PETSC_MAX_PATH_LEN];
6190     PetscInt cone[8];
6191     int      vbuf[8], v;
6192 
6193     for (c = 0; c < Ncn; ++c) {
6194       format[c * 3 + 0] = '%';
6195       format[c * 3 + 1] = 'd';
6196       format[c * 3 + 2] = ' ';
6197     }
6198     format[Ncn * 3 - 1] = '\0';
6199     for (c = 0; c < Nc; ++c) PetscCall(DMPlexSetConeSize(*dm, c, Ncn));
6200     PetscCall(DMSetUp(*dm));
6201     for (c = 0; c < Nc; ++c) {
6202       PetscCall(PetscViewerRead(viewer, line, Ncn, NULL, PETSC_STRING));
6203       switch (Ncn) {
6204       case 2:
6205         snum = sscanf(line, format, &vbuf[0], &vbuf[1]);
6206         break;
6207       case 3:
6208         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2]);
6209         break;
6210       case 4:
6211         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3]);
6212         break;
6213       case 6:
6214         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5]);
6215         break;
6216       case 8:
6217         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);
6218         break;
6219       default:
6220         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No cell shape with %d vertices", Ncn);
6221       }
6222       PetscCheck(snum == Ncn, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
6223       for (v = 0; v < Ncn; ++v) cone[v] = vbuf[v] + Nc;
6224       /* Hexahedra are inverted */
6225       if (Ncn == 8) {
6226         PetscInt tmp = cone[1];
6227         cone[1]      = cone[3];
6228         cone[3]      = tmp;
6229       }
6230       PetscCall(DMPlexSetCone(*dm, c, cone));
6231     }
6232   }
6233   PetscCall(DMPlexSymmetrize(*dm));
6234   PetscCall(DMPlexStratify(*dm));
6235   /* Read coordinates */
6236   PetscCall(DMGetCoordinateSection(*dm, &coordSection));
6237   PetscCall(PetscSectionSetNumFields(coordSection, 1));
6238   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
6239   PetscCall(PetscSectionSetChart(coordSection, Nc, Nc + Nv));
6240   for (v = Nc; v < Nc + Nv; ++v) {
6241     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
6242     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
6243   }
6244   PetscCall(PetscSectionSetUp(coordSection));
6245   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
6246   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
6247   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
6248   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
6249   PetscCall(VecSetBlockSize(coordinates, cdim));
6250   PetscCall(VecSetType(coordinates, VECSTANDARD));
6251   PetscCall(VecGetArray(coordinates, &coords));
6252   if (rank == 0) {
6253     char   format[PETSC_MAX_PATH_LEN];
6254     double x[3];
6255     int    l, val[3];
6256 
6257     if (Nl) {
6258       for (l = 0; l < Nl; ++l) {
6259         format[l * 3 + 0] = '%';
6260         format[l * 3 + 1] = 'd';
6261         format[l * 3 + 2] = ' ';
6262       }
6263       format[Nl * 3 - 1] = '\0';
6264       PetscCall(DMCreateLabel(*dm, "marker"));
6265       PetscCall(DMGetLabel(*dm, "marker", &marker));
6266     }
6267     for (v = 0; v < Nv; ++v) {
6268       PetscCall(PetscViewerRead(viewer, line, 3 + Nl, NULL, PETSC_STRING));
6269       snum = sscanf(line, "%lg %lg %lg", &x[0], &x[1], &x[2]);
6270       PetscCheck(snum == 3, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
6271       switch (Nl) {
6272       case 0:
6273         snum = 0;
6274         break;
6275       case 1:
6276         snum = sscanf(line, format, &val[0]);
6277         break;
6278       case 2:
6279         snum = sscanf(line, format, &val[0], &val[1]);
6280         break;
6281       case 3:
6282         snum = sscanf(line, format, &val[0], &val[1], &val[2]);
6283         break;
6284       default:
6285         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Request support for %d labels", Nl);
6286       }
6287       PetscCheck(snum == Nl, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
6288       for (d = 0; d < cdim; ++d) coords[v * cdim + d] = x[d];
6289       for (l = 0; l < Nl; ++l) PetscCall(DMLabelSetValue(marker, v + Nc, val[l]));
6290     }
6291   }
6292   PetscCall(VecRestoreArray(coordinates, &coords));
6293   PetscCall(DMSetCoordinatesLocal(*dm, coordinates));
6294   PetscCall(VecDestroy(&coordinates));
6295   PetscCall(PetscViewerDestroy(&viewer));
6296   if (interpolate) {
6297     DM      idm;
6298     DMLabel bdlabel;
6299 
6300     PetscCall(DMPlexInterpolate(*dm, &idm));
6301     PetscCall(DMDestroy(dm));
6302     *dm = idm;
6303 
6304     if (!Nl) {
6305       PetscCall(DMCreateLabel(*dm, "marker"));
6306       PetscCall(DMGetLabel(*dm, "marker", &bdlabel));
6307       PetscCall(DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel));
6308       PetscCall(DMPlexLabelComplete(*dm, bdlabel));
6309     }
6310   }
6311   PetscFunctionReturn(PETSC_SUCCESS);
6312 }
6313 
6314 /*@
6315   DMPlexCreateFromFile - This takes a filename and produces a `DM`
6316 
6317   Collective
6318 
6319   Input Parameters:
6320 + comm        - The communicator
6321 . filename    - A file name
6322 . plexname    - The object name of the resulting `DM`, also used for intra-datafile lookup by some formats
6323 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
6324 
6325   Output Parameter:
6326 . dm - The `DM`
6327 
6328   Options Database Key:
6329 . -dm_plex_create_from_hdf5_xdmf - use the `PETSC_VIEWER_HDF5_XDMF` format for reading HDF5
6330 
6331   Use `-dm_plex_create_ prefix` to pass options to the internal `PetscViewer`, e.g.
6332 $ -dm_plex_create_viewer_hdf5_collective
6333 
6334   Level: beginner
6335 
6336   Notes:
6337   Using `PETSCVIEWERHDF5` type with `PETSC_VIEWER_HDF5_PETSC` format, one can save multiple `DMPLEX`
6338   meshes in a single HDF5 file. This in turn requires one to name the `DMPLEX` object with `PetscObjectSetName()`
6339   before saving it with `DMView()` and before loading it with `DMLoad()` for identification of the mesh object.
6340   The input parameter name is thus used to name the `DMPLEX` object when `DMPlexCreateFromFile()` internally
6341   calls `DMLoad()`. Currently, name is ignored for other viewer types and/or formats.
6342 
6343 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`, `PetscObjectSetName()`, `DMView()`, `DMLoad()`
6344 @*/
6345 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], const char plexname[], PetscBool interpolate, DM *dm)
6346 {
6347   const char  extGmsh[]      = ".msh";
6348   const char  extGmsh2[]     = ".msh2";
6349   const char  extGmsh4[]     = ".msh4";
6350   const char  extCGNS[]      = ".cgns";
6351   const char  extExodus[]    = ".exo";
6352   const char  extExodus_e[]  = ".e";
6353   const char  extGenesis[]   = ".gen";
6354   const char  extFluent[]    = ".cas";
6355   const char  extHDF5[]      = ".h5";
6356   const char  extXDMFHDF5[]  = ".xdmf.h5";
6357   const char  extPLY[]       = ".ply";
6358   const char  extEGADSLite[] = ".egadslite";
6359   const char  extEGADS[]     = ".egads";
6360   const char  extIGES[]      = ".igs";
6361   const char  extSTEP[]      = ".stp";
6362   const char  extCV[]        = ".dat";
6363   size_t      len;
6364   PetscBool   isGmsh, isGmsh2, isGmsh4, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isPLY, isEGADSLite, isEGADS, isIGES, isSTEP, isCV, isXDMFHDF5;
6365   PetscMPIInt rank;
6366 
6367   PetscFunctionBegin;
6368   PetscAssertPointer(filename, 2);
6369   if (plexname) PetscAssertPointer(plexname, 3);
6370   PetscAssertPointer(dm, 5);
6371   PetscCall(DMInitializePackage());
6372   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromFile, 0, 0, 0, 0));
6373   PetscCallMPI(MPI_Comm_rank(comm, &rank));
6374   PetscCall(PetscStrlen(filename, &len));
6375   PetscCheck(len, comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
6376 
6377 #define CheckExtension(extension__, is_extension__) \
6378   do { \
6379     PetscAssert(sizeof(extension__), comm, PETSC_ERR_PLIB, "Zero-size extension: %s", extension__); \
6380     /* don't count the null-terminator at the end */ \
6381     const size_t ext_len = sizeof(extension__) - 1; \
6382     if (len < ext_len) { \
6383       is_extension__ = PETSC_FALSE; \
6384     } else { \
6385       PetscCall(PetscStrncmp(filename + len - ext_len, extension__, ext_len, &is_extension__)); \
6386     } \
6387   } while (0)
6388 
6389   CheckExtension(extGmsh, isGmsh);
6390   CheckExtension(extGmsh2, isGmsh2);
6391   CheckExtension(extGmsh4, isGmsh4);
6392   CheckExtension(extCGNS, isCGNS);
6393   CheckExtension(extExodus, isExodus);
6394   if (!isExodus) CheckExtension(extExodus_e, isExodus);
6395   CheckExtension(extGenesis, isGenesis);
6396   CheckExtension(extFluent, isFluent);
6397   CheckExtension(extHDF5, isHDF5);
6398   CheckExtension(extPLY, isPLY);
6399   CheckExtension(extEGADSLite, isEGADSLite);
6400   CheckExtension(extEGADS, isEGADS);
6401   CheckExtension(extIGES, isIGES);
6402   CheckExtension(extSTEP, isSTEP);
6403   CheckExtension(extCV, isCV);
6404   CheckExtension(extXDMFHDF5, isXDMFHDF5);
6405 
6406 #undef CheckExtension
6407 
6408   if (isGmsh || isGmsh2 || isGmsh4) {
6409     PetscCall(DMPlexCreateGmshFromFile(comm, filename, interpolate, dm));
6410   } else if (isCGNS) {
6411     PetscCall(DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm));
6412   } else if (isExodus || isGenesis) {
6413     PetscCall(DMPlexCreateExodusFromFile(comm, filename, interpolate, dm));
6414   } else if (isFluent) {
6415     PetscCall(DMPlexCreateFluentFromFile(comm, filename, interpolate, dm));
6416   } else if (isHDF5) {
6417     PetscViewer viewer;
6418 
6419     /* PETSC_VIEWER_HDF5_XDMF is used if the filename ends with .xdmf.h5, or if -dm_plex_create_from_hdf5_xdmf option is present */
6420     PetscCall(PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &isXDMFHDF5, NULL));
6421     PetscCall(PetscViewerCreate(comm, &viewer));
6422     PetscCall(PetscViewerSetType(viewer, PETSCVIEWERHDF5));
6423     PetscCall(PetscViewerSetOptionsPrefix(viewer, "dm_plex_create_"));
6424     PetscCall(PetscViewerSetFromOptions(viewer));
6425     PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
6426     PetscCall(PetscViewerFileSetName(viewer, filename));
6427 
6428     PetscCall(DMCreate(comm, dm));
6429     PetscCall(PetscObjectSetName((PetscObject)*dm, plexname));
6430     PetscCall(DMSetType(*dm, DMPLEX));
6431     if (isXDMFHDF5) PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF));
6432     PetscCall(DMLoad(*dm, viewer));
6433     if (isXDMFHDF5) PetscCall(PetscViewerPopFormat(viewer));
6434     PetscCall(PetscViewerDestroy(&viewer));
6435 
6436     if (interpolate) {
6437       DM idm;
6438 
6439       PetscCall(DMPlexInterpolate(*dm, &idm));
6440       PetscCall(DMDestroy(dm));
6441       *dm = idm;
6442     }
6443   } else if (isPLY) {
6444     PetscCall(DMPlexCreatePLYFromFile(comm, filename, interpolate, dm));
6445   } else if (isEGADSLite || isEGADS || isIGES || isSTEP) {
6446     if (isEGADSLite) PetscCall(DMPlexCreateEGADSLiteFromFile(comm, filename, dm));
6447     else PetscCall(DMPlexCreateEGADSFromFile(comm, filename, dm));
6448     if (!interpolate) {
6449       DM udm;
6450 
6451       PetscCall(DMPlexUninterpolate(*dm, &udm));
6452       PetscCall(DMDestroy(dm));
6453       *dm = udm;
6454     }
6455   } else if (isCV) {
6456     PetscCall(DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm));
6457   } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
6458   PetscCall(PetscStrlen(plexname, &len));
6459   if (len) PetscCall(PetscObjectSetName((PetscObject)*dm, plexname));
6460   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromFile, 0, 0, 0, 0));
6461   PetscFunctionReturn(PETSC_SUCCESS);
6462 }
6463 
6464 /*@
6465   DMPlexCreateEphemeral - This takes a `DMPlexTransform` and a base `DMPlex` and produces an ephemeral `DM`, meaning one that is created on the fly in response to queries.
6466 
6467   Input Parameters:
6468 + tr     - The `DMPlexTransform`
6469 - prefix - An options prefix, or NULL
6470 
6471   Output Parameter:
6472 . dm - The `DM`
6473 
6474   Level: beginner
6475 
6476   Notes:
6477   An emphemeral mesh is one that is not stored concretely, as in the default `DMPLEX` implementation, but rather is produced on the fly in response to queries, using information from the transform and the base mesh.
6478 
6479 .seealso: `DMPlexCreateFromFile`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
6480 @*/
6481 PetscErrorCode DMPlexCreateEphemeral(DMPlexTransform tr, const char prefix[], DM *dm)
6482 {
6483   DM           bdm, bcdm, cdm;
6484   Vec          coordinates, coordinatesNew;
6485   PetscSection cs;
6486   PetscInt     cdim, Nl;
6487 
6488   PetscFunctionBegin;
6489   PetscCall(DMCreate(PetscObjectComm((PetscObject)tr), dm));
6490   PetscCall(DMSetType(*dm, DMPLEX));
6491   ((DM_Plex *)(*dm)->data)->interpolated = DMPLEX_INTERPOLATED_FULL;
6492   // Handle coordinates
6493   PetscCall(DMPlexTransformGetDM(tr, &bdm));
6494   PetscCall(DMPlexTransformSetDimensions(tr, bdm, *dm));
6495   PetscCall(DMGetCoordinateDim(*dm, &cdim));
6496   PetscCall(DMGetCoordinateDM(bdm, &bcdm));
6497   PetscCall(DMGetCoordinateDM(*dm, &cdm));
6498   PetscCall(DMCopyDisc(bcdm, cdm));
6499   PetscCall(DMGetLocalSection(cdm, &cs));
6500   PetscCall(PetscSectionSetNumFields(cs, 1));
6501   PetscCall(PetscSectionSetFieldComponents(cs, 0, cdim));
6502   PetscCall(DMGetCoordinatesLocal(bdm, &coordinates));
6503   PetscCall(VecDuplicate(coordinates, &coordinatesNew));
6504   PetscCall(VecCopy(coordinates, coordinatesNew));
6505   PetscCall(DMSetCoordinatesLocal(*dm, coordinatesNew));
6506   PetscCall(VecDestroy(&coordinatesNew));
6507 
6508   PetscCall(PetscObjectReference((PetscObject)tr));
6509   PetscCall(DMPlexTransformDestroy(&((DM_Plex *)(*dm)->data)->tr));
6510   ((DM_Plex *)(*dm)->data)->tr = tr;
6511   PetscCall(DMPlexDistributeSetDefault(*dm, PETSC_FALSE));
6512   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)*dm, prefix));
6513   PetscCall(DMSetFromOptions(*dm));
6514 
6515   PetscCall(DMGetNumLabels(bdm, &Nl));
6516   for (PetscInt l = 0; l < Nl; ++l) {
6517     DMLabel     label, labelNew;
6518     const char *lname;
6519     PetscBool   isDepth, isCellType;
6520 
6521     PetscCall(DMGetLabelName(bdm, l, &lname));
6522     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
6523     if (isDepth) continue;
6524     PetscCall(PetscStrcmp(lname, "celltype", &isCellType));
6525     if (isCellType) continue;
6526     PetscCall(DMCreateLabel(*dm, lname));
6527     PetscCall(DMGetLabel(bdm, lname, &label));
6528     PetscCall(DMGetLabel(*dm, lname, &labelNew));
6529     PetscCall(DMLabelSetType(labelNew, DMLABELEPHEMERAL));
6530     PetscCall(DMLabelEphemeralSetLabel(labelNew, label));
6531     PetscCall(DMLabelEphemeralSetTransform(labelNew, tr));
6532     PetscCall(DMLabelSetUp(labelNew));
6533   }
6534   PetscFunctionReturn(PETSC_SUCCESS);
6535 }
6536